-
-
Notifications
You must be signed in to change notification settings - Fork 238
adding json_table function
#1142
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 10 commits
Commits
Show all changes
38 commits
Select commit
Hold shift + click to select a range
2ac0b17
starting on jsonpath logic
07d307b
:(
0a89181
trying out jsonpath parsing package
da2958b
[ga-format-pr] Run ./format_repo.sh to fix formatting
jcor11599 5a3e53e
make new JSONTable node
db01268
adding partitions, and starting to create in-memory table
9a687c9
merge
23aeb31
Merge branch 'main' into james/json_table
2982d30
working
19477b5
fixing todo
f1314c8
Merge branch 'main' into james/json_table
855b347
Merge branch 'main' into james/json_table
a51c1db
Merge branch 'main' into james/json_table
6d93e54
trying stuff
5390ab1
Merge branch 'main' into james/json_table
9c2f480
working so far
2f76cb2
parse expr
cfb9a6a
bump
5dfcc74
moving sqlparser code out of plan
05f39a7
move logic to rowiter
ab28f59
adding a bad test
03cedc8
a
50dedb6
a
47d4512
Merge branch 'main' into james/json_table
040422d
i did this yesterday and it didn't work, today is a different day
5b51930
removing unnecessary setup
01de00d
bumping
31d42ea
fix comment
c28fee8
more tests
bd44c84
adding column test
55dfbd3
merge with main
a9e1cdf
passing in row
a758d4d
removing comment
56dce6e
Merge branch 'main' into james/json_table
f4966c3
moving logic into next
07b674a
avoid precision problem
80aca34
adding test for null
fd72a9e
use single quote instead
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,181 @@ | ||
| // Copyright 2020-2021 Dolthub, Inc. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package plan | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "io" | ||
|
|
||
| "github.com/dolthub/vitess/go/vt/sqlparser" | ||
| "github.com/oliveagle/jsonpath" | ||
|
|
||
| "github.com/dolthub/go-mysql-server/sql" | ||
| ) | ||
|
|
||
| type jsonTablePartition struct { | ||
| key []byte | ||
| } | ||
|
|
||
| var _ sql.Partition = &jsonTablePartition{} | ||
|
|
||
| func (j *jsonTablePartition) Key() []byte { | ||
| return j.key | ||
| } | ||
|
|
||
| type jsonTablePartitionIter struct { | ||
| keys [][]byte | ||
| pos int | ||
| } | ||
|
|
||
| var _ sql.PartitionIter = &jsonTablePartitionIter{} | ||
|
|
||
| func (j *jsonTablePartitionIter) Close(ctx *sql.Context) error { | ||
| return nil | ||
| } | ||
|
|
||
| func (j *jsonTablePartitionIter) Next(ctx *sql.Context) (sql.Partition, error) { | ||
| if j.pos >= len(j.keys) { | ||
| return nil, io.EOF | ||
| } | ||
|
|
||
| key := j.keys[j.pos] | ||
| j.pos++ | ||
| return &jsonTablePartition{key}, nil | ||
| } | ||
|
|
||
| type jsonTableRowIter struct { | ||
| rows []sql.Row | ||
| pos int | ||
| } | ||
|
|
||
| var _ sql.RowIter = &jsonTableRowIter{} | ||
|
|
||
| func (j *jsonTableRowIter) Next(ctx *sql.Context) (sql.Row, error) { | ||
| if j.pos >= len(j.rows) { | ||
| return nil, io.EOF | ||
| } | ||
| row := j.rows[j.pos] | ||
| j.pos++ | ||
| return row, nil | ||
| } | ||
|
|
||
| func (j *jsonTableRowIter) Close(ctx *sql.Context) error { | ||
| return nil | ||
| } | ||
|
|
||
| type JSONTable struct { | ||
| name string | ||
| schema sql.PrimaryKeySchema | ||
| data []sql.Row | ||
| rowIdx uint64 | ||
| } | ||
|
|
||
| var _ sql.Table = &JSONTable{} | ||
| var _ sql.Node = &JSONTable{} | ||
|
|
||
| // Name implements the sql.Table interface | ||
| func (t *JSONTable) Name() string { | ||
| return t.name | ||
| } | ||
|
|
||
| // String implements the sql.Table interface | ||
| func (t *JSONTable) String() string { | ||
| return t.name | ||
| } | ||
|
|
||
| // Schema implements the sql.Table interface | ||
| func (t *JSONTable) Schema() sql.Schema { | ||
| return t.schema.Schema | ||
| } | ||
|
|
||
| // Partitions implements the sql.Table interface | ||
| func (t *JSONTable) Partitions(ctx *sql.Context) (sql.PartitionIter, error) { | ||
| // TODO: this does nothing | ||
| return &jsonTablePartitionIter{ | ||
| keys: [][]byte{{0}}, | ||
| }, nil | ||
| } | ||
|
|
||
| // PartitionRows implements the sql.Table interface | ||
| func (t *JSONTable) PartitionRows(ctx *sql.Context, partition sql.Partition) (sql.RowIter, error) { | ||
| return &jsonTableRowIter{ | ||
| rows: t.data, | ||
| }, nil | ||
| } | ||
|
|
||
| func (t *JSONTable) Resolved() bool { | ||
| return true | ||
| } | ||
|
|
||
| func (t *JSONTable) Children() []sql.Node { | ||
| return nil | ||
| } | ||
|
|
||
| func (t *JSONTable) RowIter(ctx *sql.Context, row sql.Row) (sql.RowIter, error) { | ||
| return nil, nil | ||
| } | ||
|
|
||
| func (t *JSONTable) WithChildren(children ...sql.Node) (sql.Node, error) { | ||
| return t, nil | ||
| } | ||
|
|
||
| func (t *JSONTable) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool { | ||
| return true | ||
| } | ||
|
|
||
| // NewJSONTable creates a new in memory table from the JSON formatted data, a jsonpath path string, and table spec. | ||
| func NewJSONTable(data []byte, path string, spec *sqlparser.TableSpec, alias sqlparser.TableIdent, schema sql.PrimaryKeySchema) (sql.Node, error) { | ||
|
||
| // Parse data as JSON | ||
| var jsonData interface{} | ||
| if err := json.Unmarshal(data, &jsonData); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // Get data specified from initial path | ||
| jsonPathData, err := jsonpath.JsonPathLookup(jsonData, path) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // Create new JSONTable node | ||
| table := &JSONTable{ | ||
| name: alias.String(), | ||
| schema: schema, | ||
| } | ||
|
|
||
| // Allocate number of rows, based off values in first column | ||
| if v, err := jsonpath.JsonPathLookup(jsonPathData, spec.Columns[0].Type.Path); err == nil { | ||
| if val, ok := v.([]interface{}); ok { | ||
| table.data = make([]sql.Row, len(val)) | ||
| } | ||
| } | ||
|
|
||
| // Fill in table with data | ||
| for _, col := range spec.Columns { | ||
| v, err := jsonpath.JsonPathLookup(jsonPathData, col.Type.Path) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| colData, ok := v.([]interface{}) | ||
| if !ok { | ||
| panic("TODO: good error message") | ||
| } | ||
| for i, val := range colData { | ||
| table.data[i] = append(table.data[i], val) | ||
| } | ||
| } | ||
|
|
||
| return table, nil | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You might consider modelling this as a sql.TableFunction instead
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I did originally, but I had trouble implementing the
NewInstancemethod