diff --git a/duck/data/arrow.go b/duck/data/arrow.go deleted file mode 100644 index a42465f..0000000 --- a/duck/data/arrow.go +++ /dev/null @@ -1,198 +0,0 @@ -package data - -import ( - "encoding/json" - "fmt" - - "github.com/apache/arrow/go/v15/arrow" - "github.com/grafana/grafana-plugin-sdk-go/data" -) - -// keys added to arrow field metadata -const metadataKeyName = "name" // standard property -const metadataKeyConfig = "config" // FieldConfig serialized as JSON -const metadataKeyLabels = "labels" // labels serialized as JSON -const metadataKeyRefID = "refId" // added to the table metadata - -// MarshalArrow converts the Frame to an arrow table and returns a byte -// representation of that table. -// All fields of a Frame must be of the same length or an error is returned. -func MarshalArrow(f *data.Frame) (*arrow.Schema, error) { - if _, err := f.RowLen(); err != nil { - logger.Error("failed to get row length", "error", err) - return nil, err - } - - arrowFields, err := buildArrowFields(f) - if err != nil { - logger.Error("failed to build arrow fields", "error", err) - return nil, err - } - - schema, err := buildArrowSchema(f, arrowFields) - if err != nil { - logger.Error("failed to build arrow schema", "error", err) - return nil, err - } - - return schema, nil -} - -// buildArrowFields builds Arrow field definitions from a Frame. -func buildArrowFields(f *data.Frame) ([]arrow.Field, error) { - arrowFields := make([]arrow.Field, len(f.Fields)) - - for i, field := range f.Fields { - t, nullable, err := fieldToArrow(field) - if err != nil { - return nil, err - } - fieldMeta := map[string]string{} - - if field.Labels != nil { - if fieldMeta[metadataKeyLabels], err = toJSONString(field.Labels); err != nil { - logger.Error("failed to serialize labels", "error", err) - return nil, err - } - } - - name := getFieldName(field) - if field.Config != nil { - str, err := toJSONString(field.Config) - if err != nil { - logger.Error("failed to serialize field config", "error", err) - return nil, err - } - fieldMeta[metadataKeyConfig] = str - } - - arrowFields[i] = arrow.Field{ - Name: name, - Type: t, - Metadata: arrow.MetadataFrom(fieldMeta), - Nullable: nullable, - } - } - - return arrowFields, nil -} - -// buildArrowSchema builds an Arrow schema for a Frame. -func buildArrowSchema(f *data.Frame, fs []arrow.Field) (*arrow.Schema, error) { - tableMetaMap := map[string]string{ - metadataKeyName: f.Name, - metadataKeyRefID: f.RefID, - } - if f.Meta != nil { - str, err := toJSONString(f.Meta) - if err != nil { - logger.Error("failed to serialize frame meta", "error", err) - return nil, err - } - tableMetaMap["meta"] = str - } - tableMeta := arrow.MetadataFrom(tableMetaMap) - - return arrow.NewSchema(fs, &tableMeta), nil -} - -// fieldToArrow returns the corresponding Arrow primitive type and nullable property to the fields' -// Vector primitives. -// nolint:gocyclo -func fieldToArrow(f *data.Field) (arrow.DataType, bool, error) { - switch f.Type() { - case data.FieldTypeString: - return &arrow.StringType{}, false, nil - case data.FieldTypeNullableString: - return &arrow.StringType{}, true, nil - - // Ints - case data.FieldTypeInt8: - return &arrow.Int8Type{}, false, nil - case data.FieldTypeNullableInt8: - return &arrow.Int8Type{}, true, nil - - case data.FieldTypeInt16: - return &arrow.Int16Type{}, false, nil - case data.FieldTypeNullableInt16: - return &arrow.Int16Type{}, true, nil - - case data.FieldTypeInt32: - return &arrow.Int32Type{}, false, nil - case data.FieldTypeNullableInt32: - return &arrow.Int32Type{}, true, nil - - case data.FieldTypeInt64: - return &arrow.Int64Type{}, false, nil - case data.FieldTypeNullableInt64: - return &arrow.Int64Type{}, true, nil - - // Uints - case data.FieldTypeUint8: - return &arrow.Uint8Type{}, false, nil - case data.FieldTypeNullableUint8: - return &arrow.Uint8Type{}, true, nil - - case data.FieldTypeUint16, data.FieldTypeEnum: - return &arrow.Uint16Type{}, false, nil - case data.FieldTypeNullableUint16, data.FieldTypeNullableEnum: - return &arrow.Uint16Type{}, true, nil - - case data.FieldTypeUint32: - return &arrow.Uint32Type{}, false, nil - case data.FieldTypeNullableUint32: - return &arrow.Uint32Type{}, true, nil - - case data.FieldTypeUint64: - return &arrow.Uint64Type{}, false, nil - case data.FieldTypeNullableUint64: - return &arrow.Uint64Type{}, true, nil - - case data.FieldTypeFloat32: - return &arrow.Float32Type{}, false, nil - case data.FieldTypeNullableFloat32: - return &arrow.Float32Type{}, true, nil - - case data.FieldTypeFloat64: - return &arrow.Float64Type{}, false, nil - case data.FieldTypeNullableFloat64: - return &arrow.Float64Type{}, true, nil - - case data.FieldTypeBool: - return &arrow.BooleanType{}, false, nil - case data.FieldTypeNullableBool: - return &arrow.BooleanType{}, true, nil - - case data.FieldTypeTime: - return &arrow.TimestampType{Unit: arrow.Nanosecond}, false, nil - case data.FieldTypeNullableTime: - return &arrow.TimestampType{Unit: arrow.Nanosecond}, true, nil - - case data.FieldTypeJSON: - return &arrow.BinaryType{}, false, nil - case data.FieldTypeNullableJSON: - return &arrow.BinaryType{}, true, nil - - default: - logger.Error("unsupported type for conversion to arrow", "type", f.Type()) - return nil, false, fmt.Errorf("unsupported type for conversion to arrow: %T", f.Type()) - } -} - -// ToJSONString calls json.Marshal on val and returns it as a string. An -// error is returned if json.Marshal errors. -func toJSONString(val interface{}) (string, error) { - b, err := json.Marshal(val) - if err != nil { - logger.Error("failed to marshal value to json", "error", err) - return "", err - } - return string(b), nil -} - -func getFieldName(field *data.Field) string { - if field.Config != nil && field.Config.DisplayName != "" { - return field.Config.DisplayName - } - return field.Name -} diff --git a/duck/data/parquet.go b/duck/data/parquet.go index d0763cd..ef435a7 100644 --- a/duck/data/parquet.go +++ b/duck/data/parquet.go @@ -49,12 +49,15 @@ func ToParquet(frames []*data.Frame, chunk int) (map[string]string, error) { for i, frame := range frameList { dirs[frame.RefID] = dir - schema, err := MarshalArrow(frame) + table, err := data.FrameToArrowTable(frame) if err != nil { - logger.Error("failed to marshal arrow schema", "error", err) + logger.Error("failed to create arrow table", "error", err) return nil, err } + // TODO... no need to create the table from data anymore, + // BUT it means any modifications must be made before creating the arrow.Table + schema := table.Schema() data := frameData(frame) if chunk > 0 { @@ -124,6 +127,13 @@ func frameData(frame *data.Frame) FrameData { return data } +func getFieldName(field *data.Field) string { + if field.Config != nil && field.Config.DisplayName != "" { + return field.Config.DisplayName + } + return field.Name +} + func write(dir string, name string, schema *arrow.Schema, jsonData []byte) (string, string, error) { filename := path.Join(dir, name+".parquet") output, err := os.Create(filename) diff --git a/go.mod b/go.mod index f4d4238..33a1d30 100644 --- a/go.mod +++ b/go.mod @@ -2,19 +2,28 @@ module github.com/scottlepp/go-duck go 1.21 +require ( + github.com/apache/arrow/go/v15 v15.0.2 + github.com/grafana/grafana-plugin-sdk-go v0.234.0 + github.com/stretchr/testify v1.9.0 +) + require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/andybalholm/brotli v1.0.5 // indirect - github.com/apache/arrow/go/v15 v15.0.2 github.com/cheekybits/genny v1.0.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/mattetti/filebuffer v1.0.1 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.10 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect @@ -22,18 +31,18 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.1.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.25.0 // indirect - golang.org/x/sync v0.6.0 // indirect + golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect - google.golang.org/grpc v1.63.2 // indirect + google.golang.org/grpc v1.64.0 // indirect google.golang.org/protobuf v1.33.0 // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -42,17 +51,7 @@ require ( github.com/apache/thrift v0.17.0 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/golang/snappy v0.0.4 // indirect - github.com/grafana/grafana-plugin-sdk-go v0.231.0 github.com/klauspost/compress v1.16.7 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect - github.com/stretchr/testify v1.9.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect ) - -require ( - github.com/fatih/color v1.15.0 // indirect - github.com/hashicorp/go-hclog v1.6.3 // indirect - github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect - github.com/rivo/uniseg v0.1.0 // indirect -) diff --git a/go.sum b/go.sum index c0f01d6..ffeb110 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/grafana/grafana-plugin-sdk-go v0.231.0 h1:Qt4PBDR8b4MTUxL48EaZw1fHI1rXUNNhvTU/Nf0Ex2g= -github.com/grafana/grafana-plugin-sdk-go v0.231.0/go.mod h1:8fJk+5J1hMkpqY/7vrXHKgAsqELWNkQvLQ5A5xCVZHk= +github.com/grafana/grafana-plugin-sdk-go v0.234.0 h1:p20XfGKB3Z/8aZ6jut+FIU/0cXw+dLkcGFnxJbyFd+k= +github.com/grafana/grafana-plugin-sdk-go v0.234.0/go.mod h1:FlXjmBESxaD6Hoi8ojWLkH007nyjtJM3XC8SpwzF/YE= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 h1:qnpSQwGEnkcRpTqNOIR6bJbR0gAorgP9CSALpRcKoAA= github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1/go.mod h1:lXGCsh6c22WGtjr+qGHj1otzZpV/1kwTMAqkwZsnWRU= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 h1:pRhl55Yx1eC7BZ1N+BBWwnKaMyD8uC+34TLdndZMAKk= @@ -86,12 +86,6 @@ github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGC github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magefile/mage v1.15.0 h1:BvGheCMAsG3bWUDbZ8AyXXpCNwU9u5CB6sM+HNb9HYg= github.com/magefile/mage v1.15.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -195,8 +189,8 @@ golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -216,18 +210,16 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3j golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be h1:Zz7rLWqp0ApfsR/l7+zSHhY3PMiH2xqgxlfYfAfNpoU= google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be h1:LG9vZxsWGOmUKieR8wPAUR3u3MpnYFQZROPIMaXh7/A= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= -google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= -google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=