Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
2ac0b17
starting on jsonpath logic
Jul 29, 2022
07d307b
:(
Jul 29, 2022
0a89181
trying out jsonpath parsing package
Jul 31, 2022
da2958b
[ga-format-pr] Run ./format_repo.sh to fix formatting
jcor11599 Jul 31, 2022
5a3e53e
make new JSONTable node
Aug 1, 2022
db01268
adding partitions, and starting to create in-memory table
Aug 1, 2022
9a687c9
merge
Aug 1, 2022
23aeb31
Merge branch 'main' into james/json_table
Aug 2, 2022
2982d30
working
Aug 2, 2022
19477b5
fixing todo
Aug 2, 2022
f1314c8
Merge branch 'main' into james/json_table
Aug 2, 2022
855b347
Merge branch 'main' into james/json_table
Aug 8, 2022
a51c1db
Merge branch 'main' into james/json_table
Aug 10, 2022
6d93e54
trying stuff
Aug 11, 2022
5390ab1
Merge branch 'main' into james/json_table
Aug 11, 2022
9c2f480
working so far
Aug 11, 2022
2f76cb2
parse expr
Aug 11, 2022
cfb9a6a
bump
Aug 11, 2022
5dfcc74
moving sqlparser code out of plan
Aug 12, 2022
05f39a7
move logic to rowiter
Aug 12, 2022
ab28f59
adding a bad test
Aug 12, 2022
03cedc8
a
Aug 12, 2022
50dedb6
a
Aug 12, 2022
47d4512
Merge branch 'main' into james/json_table
Aug 14, 2022
040422d
i did this yesterday and it didn't work, today is a different day
Aug 14, 2022
5b51930
removing unnecessary setup
Aug 15, 2022
01de00d
bumping
Aug 15, 2022
31d42ea
fix comment
Aug 15, 2022
c28fee8
more tests
Aug 15, 2022
bd44c84
adding column test
Aug 15, 2022
55dfbd3
merge with main
Aug 15, 2022
a9e1cdf
passing in row
Aug 15, 2022
a758d4d
removing comment
Aug 16, 2022
56dce6e
Merge branch 'main' into james/json_table
Aug 16, 2022
f4966c3
moving logic into next
Aug 16, 2022
07b674a
avoid precision problem
Aug 16, 2022
80aca34
adding test for null
Aug 16, 2022
fd72a9e
use single quote instead
Aug 16, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions sql/parse/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -2556,6 +2556,9 @@ func tableExprToTable(
case *sqlparser.JoinTableExpr:
return joinTableExpr(ctx, t)

case *sqlparser.JSONTableExpr:
return jsonTableExpr(ctx, t)

case *sqlparser.ParenTableExpr:
if len(t.Exprs) == 1 {
switch j := t.Exprs[0].(type) {
Expand Down Expand Up @@ -2612,6 +2615,16 @@ func joinTableExpr(ctx *sql.Context, t *sqlparser.JoinTableExpr) (sql.Node, erro
}
}

func jsonTableExpr(ctx *sql.Context, t *sqlparser.JSONTableExpr) (sql.Node, error) {
// TODO: figure out how to deal with nested
// TODO: figure out how to deal with table alias
sch, err := TableSpecToSchema(ctx, t.Columns)
if err != nil {
return nil, err
}
return plan.NewJSONTable(t.Data, t.Path, t.Columns, t.Alias, sch)
}

func whereToFilter(ctx *sql.Context, w *sqlparser.Where, child sql.Node) (*plan.Filter, error) {
c, err := ExprToExpression(ctx, w.Expr)
if err != nil {
Expand Down
181 changes: 181 additions & 0 deletions sql/plan/json_table.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
// Copyright 2020-2021 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package plan

import (
"encoding/json"
"io"

"github.com/dolthub/vitess/go/vt/sqlparser"
"github.com/oliveagle/jsonpath"

"github.com/dolthub/go-mysql-server/sql"
)

type jsonTablePartition struct {
key []byte
}

var _ sql.Partition = &jsonTablePartition{}

func (j *jsonTablePartition) Key() []byte {
return j.key
}

type jsonTablePartitionIter struct {
keys [][]byte
pos int
}

var _ sql.PartitionIter = &jsonTablePartitionIter{}

func (j *jsonTablePartitionIter) Close(ctx *sql.Context) error {
return nil
}

func (j *jsonTablePartitionIter) Next(ctx *sql.Context) (sql.Partition, error) {
if j.pos >= len(j.keys) {
return nil, io.EOF
}

key := j.keys[j.pos]
j.pos++
return &jsonTablePartition{key}, nil
}

type jsonTableRowIter struct {
rows []sql.Row
pos int
}

var _ sql.RowIter = &jsonTableRowIter{}

func (j *jsonTableRowIter) Next(ctx *sql.Context) (sql.Row, error) {
if j.pos >= len(j.rows) {
return nil, io.EOF
}
row := j.rows[j.pos]
j.pos++
return row, nil
}

func (j *jsonTableRowIter) Close(ctx *sql.Context) error {
return nil
}

type JSONTable struct {
name string
schema sql.PrimaryKeySchema
data []sql.Row
rowIdx uint64
}

var _ sql.Table = &JSONTable{}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You might consider modelling this as a sql.TableFunction instead

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did originally, but I had trouble implementing the NewInstance method

var _ sql.Node = &JSONTable{}

// Name implements the sql.Table interface
func (t *JSONTable) Name() string {
return t.name
}

// String implements the sql.Table interface
func (t *JSONTable) String() string {
return t.name
}

// Schema implements the sql.Table interface
func (t *JSONTable) Schema() sql.Schema {
return t.schema.Schema
}

// Partitions implements the sql.Table interface
func (t *JSONTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) {
// TODO: this does nothing
return &jsonTablePartitionIter{
keys: [][]byte{{0}},
}, nil
}

// PartitionRows implements the sql.Table interface
func (t *JSONTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) {
return &jsonTableRowIter{
rows: t.data,
}, nil
}

func (t *JSONTable) Resolved() bool {
return true
}

func (t *JSONTable) Children() []sql.Node {
return nil
}

func (t *JSONTable) RowIter(ctx *sql.Context, row sql.Row) (sql.RowIter, error) {
return nil, nil
}

func (t *JSONTable) WithChildren(children ...sql.Node) (sql.Node, error) {
return t, nil
}

func (t *JSONTable) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool {
return true
}

// NewJSONTable creates a new in memory table from the JSON formatted data, a jsonpath path string, and table spec.
func NewJSONTable(data []byte, path string, spec *sqlparser.TableSpec, alias sqlparser.TableIdent, schema sql.PrimaryKeySchema) (sql.Node, error) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As in the other PR for vitess, this needs to take an expression for the JSON data, not a []byte

Also sqlparser.* should not make it down to this layer of the code, all that translation should be handled in parse.go

// Parse data as JSON
var jsonData interface{}
if err := json.Unmarshal(data, &jsonData); err != nil {
return nil, err
}

// Get data specified from initial path
jsonPathData, err := jsonpath.JsonPathLookup(jsonData, path)
if err != nil {
return nil, err
}

// Create new JSONTable node
table := &JSONTable{
name: alias.String(),
schema: schema,
}

// Allocate number of rows, based off values in first column
if v, err := jsonpath.JsonPathLookup(jsonPathData, spec.Columns[0].Type.Path); err == nil {
if val, ok := v.([]interface{}); ok {
table.data = make([]sql.Row, len(val))
}
}

// Fill in table with data
for _, col := range spec.Columns {
v, err := jsonpath.JsonPathLookup(jsonPathData, col.Type.Path)
if err != nil {
return nil, err
}
colData, ok := v.([]interface{})
if !ok {
panic("TODO: good error message")
}
for i, val := range colData {
table.data[i] = append(table.data[i], val)
}
}

return table, nil
}