diff --git a/engine/core/compile.go b/engine/core/compile.go index f3daec08..61f59762 100644 --- a/engine/core/compile.go +++ b/engine/core/compile.go @@ -17,6 +17,10 @@ type DefaultPipeline struct { markTypes map[string]gdbi.DataType } +func NewPipeline(procs []gdbi.Processor, ps *PipelineState) *DefaultPipeline { + return &DefaultPipeline{procs, ps.LastType, ps.MarkTypes} +} + // DataType return the datatype func (pipe *DefaultPipeline) DataType() gdbi.DataType { return pipe.dataType @@ -65,8 +69,8 @@ func (ps *PipelineState) StepLoadData() bool { } func NewPipelineState(stmts []*gripql.GraphStatement) *PipelineState { - steps := engine.PipelineSteps(stmts) - stepOut := engine.PipelineStepOutputs(stmts) + steps := inspect.PipelineSteps(stmts) + stepOut := inspect.PipelineStepOutputs(stmts) return &PipelineState{ LastType: gdbi.NoData, @@ -84,10 +88,13 @@ func (comp DefaultCompiler) Compile(stmts []*gripql.GraphStatement) (gdbi.Pipeli stmts = Flatten(stmts) - if err := validate(stmts); err != nil { + if err := Validate(stmts); err != nil { return &DefaultPipeline{}, fmt.Errorf("invalid statments: %s", err) } + stmts = IndexStartOptimize(stmts) + + ps := NewPipelineState(stmts) procs := make([]gdbi.Processor, 0, len(stmts)) @@ -101,8 +108,6 @@ func (comp DefaultCompiler) Compile(stmts []*gripql.GraphStatement) (gdbi.Pipeli procs = append(procs, p) } - procs = indexStartOptimize(procs) - return &DefaultPipeline{procs, ps.LastType, ps.MarkTypes}, nil } @@ -302,143 +307,21 @@ func StatementProcessor(gs *gripql.GraphStatement, db gdbi.GraphInterface, ps *P ps.LastType = gdbi.AggregationData return &aggregate{stmt.Aggregate.Aggregations}, nil + //Custom graph statements + case *gripql.GraphStatement_LookupVertsIndex: + ps.LastType = gdbi.VertexData + return &LookupVertsIndex{db:db, labels:stmt.Labels, loadData:ps.StepLoadData()}, nil + default: return nil, fmt.Errorf("unknown statement type") } } -// For V().Has(Eq("$.label", "Person")) and V().Has(Eq("$.gid", "1")) queries, streamline into a single index lookup -func indexStartOptimize(pipe []gdbi.Processor) []gdbi.Processor { - optimized := []gdbi.Processor{} - - var lookupV *LookupVerts - hasIDIdx := []int{} - hasLabelIdx := []int{} - isDone := false - for i, step := range pipe { - if isDone { - break - } - if i == 0 { - if lv, ok := step.(*LookupVerts); ok { - lookupV = lv - } else { - break - } - continue - } - switch s := step.(type) { - case *HasID: - hasIDIdx = append(hasIDIdx, i) - case *HasLabel: - hasLabelIdx = append(hasLabelIdx, i) - case *Has: - if and := s.stmt.GetAnd(); and != nil { - stmts := and.GetExpressions() - newPipe := []gdbi.Processor{} - newPipe = append(newPipe, pipe[:i]...) - for _, stmt := range stmts { - newPipe = append(newPipe, &Has{stmt: stmt}) - } - newPipe = append(newPipe, pipe[i+1:]...) - return indexStartOptimize(newPipe) - } - if cond := s.stmt.GetCondition(); cond != nil { - path := jsonpath.GetJSONPath(cond.Key) - switch path { - case "$.gid": - hasIDIdx = append(hasIDIdx, i) - case "$.label": - hasLabelIdx = append(hasLabelIdx, i) - default: - // do nothing - } - } - default: - isDone = true - } - } - - idOpt := false - if len(hasIDIdx) > 0 { - ids := []string{} - idx := hasIDIdx[0] - if has, ok := pipe[idx].(*Has); ok { - ids = append(ids, extractHasVals(has)...) - } - if has, ok := pipe[idx].(*HasID); ok { - ids = append(ids, has.ids...) - } - if len(ids) > 0 { - idOpt = true - hIdx := &LookupVerts{ids: ids, db: lookupV.db} - optimized = append(optimized, hIdx) - } - } - - labelOpt := false - if len(hasLabelIdx) > 0 && !idOpt { - labels := []string{} - idx := hasLabelIdx[0] - if has, ok := pipe[idx].(*Has); ok { - labels = append(labels, extractHasVals(has)...) - } - if has, ok := pipe[idx].(*HasLabel); ok { - labels = append(labels, has.labels...) - } - if len(labels) > 0 { - labelOpt = true - hIdx := &LookupVertsIndex{labels: labels, db: lookupV.db} - optimized = append(optimized, hIdx) - } - } - for i, step := range pipe { - if idOpt || labelOpt { - if i == 0 { - continue - } - } else { - optimized = append(optimized, step) - } - if idOpt { - if i != hasIDIdx[0] { - optimized = append(optimized, step) - } - } - if labelOpt { - if i != hasLabelIdx[0] { - optimized = append(optimized, step) - } - } - } - return optimized -} - -func extractHasVals(h *Has) []string { - vals := []string{} - if cond := h.stmt.GetCondition(); cond != nil { - // path := jsonpath.GetJSONPath(cond.Key) - val := protoutil.UnWrapValue(cond.Value) - switch cond.Condition { - case gripql.Condition_EQ: - if l, ok := val.(string); ok { - vals = []string{l} - } - case gripql.Condition_WITHIN: - v := val.([]interface{}) - for _, x := range v { - vals = append(vals, x.(string)) - } - default: - // do nothing - } - } - return vals -} -func validate(stmts []*gripql.GraphStatement) error { +//Validate checks pipeline for chains of statements that won't work +func Validate(stmts []*gripql.GraphStatement) error { for i, gs := range stmts { // Validate that the first statement is V() or E() if i == 0 { diff --git a/engine/core/optimize.go b/engine/core/optimize.go new file mode 100644 index 00000000..ea270798 --- /dev/null +++ b/engine/core/optimize.go @@ -0,0 +1,143 @@ + +package core + +import ( + "github.com/bmeg/grip/jsonpath" + "github.com/bmeg/grip/protoutil" + "github.com/bmeg/grip/gripql" +) + +//IndexStartOptimize looks at processor pipeline for queries like +// V().Has(Eq("$.label", "Person")) and V().Has(Eq("$.gid", "1")), +// streamline into a single index lookup +func IndexStartOptimize(pipe []*gripql.GraphStatement) []*gripql.GraphStatement { + optimized := []*gripql.GraphStatement{} + + //var lookupV *gripql.GraphStatement_V + hasIDIdx := []int{} + hasLabelIdx := []int{} + isDone := false + for i, step := range pipe { + if isDone { + break + } + if i == 0 { + if _, ok := step.GetStatement().(*gripql.GraphStatement_V); ok { + //lookupV = lv + } else { + break + } + continue + } + switch s := step.GetStatement().(type) { + case *gripql.GraphStatement_HasId: + hasIDIdx = append(hasIDIdx, i) + case *gripql.GraphStatement_HasLabel: + hasLabelIdx = append(hasLabelIdx, i) + case *gripql.GraphStatement_Has: + if and := s.Has.GetAnd(); and != nil { + stmts := and.GetExpressions() + newPipe := []*gripql.GraphStatement{} + newPipe = append(newPipe, pipe[:i]...) + for _, stmt := range stmts { + newPipe = append(newPipe, &gripql.GraphStatement{Statement:&gripql.GraphStatement_Has{Has: stmt}}) + } + newPipe = append(newPipe, pipe[i+1:]...) + return IndexStartOptimize(newPipe) + } + if cond := s.Has.GetCondition(); cond != nil { + path := jsonpath.GetJSONPath(cond.Key) + switch path { + case "$.gid": + hasIDIdx = append(hasIDIdx, i) + case "$.label": + hasLabelIdx = append(hasLabelIdx, i) + default: + // do nothing + } + } + default: + isDone = true + } + } + + idOpt := false + if len(hasIDIdx) > 0 { + ids := []string{} + idx := hasIDIdx[0] + if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { + ids = append(ids, extractHasVals(has)...) + } + if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_HasId); ok { + ids = append(ids, protoutil.AsStringList(has.HasId)...) + } + if len(ids) > 0 { + idOpt = true + hIdx := &gripql.GraphStatement_V{V: protoutil.AsListValue(ids)} + optimized = append(optimized, &gripql.GraphStatement{Statement:hIdx}) + } + } + + labelOpt := false + if len(hasLabelIdx) > 0 && !idOpt { + labels := []string{} + idx := hasLabelIdx[0] + if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_Has); ok { + labels = append(labels, extractHasVals(has)...) + } + if has, ok := pipe[idx].GetStatement().(*gripql.GraphStatement_HasLabel); ok { + labels = append(labels, protoutil.AsStringList(has.HasLabel)...) + } + if len(labels) > 0 { + labelOpt = true + hIdx := &gripql.GraphStatement_LookupVertsIndex{Labels: labels} + optimized = append(optimized, &gripql.GraphStatement{hIdx}) + } + } + + for i, step := range pipe { + if idOpt || labelOpt { + if i == 0 { + continue + } + } else { + optimized = append(optimized, step) + } + if idOpt { + if i != hasIDIdx[0] { + optimized = append(optimized, step) + } + } + if labelOpt { + if i != hasLabelIdx[0] { + optimized = append(optimized, step) + } + } + } + + return optimized +} + + + +func extractHasVals(h *gripql.GraphStatement_Has) []string { + vals := []string{} + if cond := h.Has.GetCondition(); cond != nil { + // path := jsonpath.GetJSONPath(cond.Key) + val := protoutil.UnWrapValue(cond.Value) + switch cond.Condition { + case gripql.Condition_EQ: + if l, ok := val.(string); ok { + vals = []string{l} + } + case gripql.Condition_WITHIN: + v := val.([]interface{}) + for _, x := range v { + vals = append(vals, x.(string)) + } + default: + // do nothing + } + } + return vals +} diff --git a/engine/core/optimizer_test.go b/engine/core/optimizer_test.go index 723f17d9..d6546856 100644 --- a/engine/core/optimizer_test.go +++ b/engine/core/optimizer_test.go @@ -4,87 +4,119 @@ import ( "reflect" "testing" - "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/gripql" + "github.com/bmeg/grip/protoutil" "github.com/davecgh/go-spew/spew" ) func TestIndexStartOptimize(t *testing.T) { - expected := []gdbi.Processor{ - &LookupVerts{ids: []string{"1", "2", "3"}}, - &LookupVertexAdjOut{}, + expected := []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{V: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original := []gdbi.Processor{ - &LookupVerts{ids: []string{"1", "2", "3"}}, - &LookupVertexAdjOut{}, + original := []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{V: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized := indexStartOptimize(original) + optimized := IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVerts{}, - &LookupVertexAdjOut{}, - &HasID{ids: []string{"1", "2", "3"}}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_HasId{HasId: protoutil.AsListValue([]string{"1", "2", "3"})}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &LookupVertexAdjOut{}, - &HasID{ids: []string{"1", "2", "3"}}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_HasId{HasId: protoutil.AsListValue([]string{"1", "2", "3"})}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVerts{ids: []string{"1", "2", "3"}}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{V: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &HasID{ids: []string{"1", "2", "3"}}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_HasId{HasId: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVerts{ids: []string{"1", "2", "3"}}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{V: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Within("_gid", "1", "2", "3")}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_gid", + Value: protoutil.WrapValue([]string{"1", "2", "3"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Neq("_gid", "1")}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_NEQ, + Key: "_gid", + Value: protoutil.WrapValue("1"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Neq("_gid", "1")}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_NEQ, + Key: "_gid", + Value: protoutil.WrapValue("1"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) @@ -92,18 +124,44 @@ func TestIndexStartOptimize(t *testing.T) { } // order shouldnt matter - expected = []gdbi.Processor{ - &LookupVerts{ids: []string{"1", "2", "3"}}, - &Has{stmt: gripql.Eq("$.data.foo", "bar")}, - &LookupVertexAdjOut{}, - } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Eq("$.data.foo", "bar")}, - &Has{stmt: gripql.Within("_gid", "1", "2", "3")}, - &LookupVertexAdjOut{}, - } - optimized = indexStartOptimize(original) + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{V: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + } + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_gid", + Value: protoutil.WrapValue([]string{"1", "2", "3"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + } + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) @@ -111,34 +169,54 @@ func TestIndexStartOptimize(t *testing.T) { } // only use the first statement - expected = []gdbi.Processor{ - &LookupVerts{ids: []string{"1", "2", "3"}}, - &Has{stmt: gripql.Within("_gid", "4", "5")}, - &LookupVertexAdjOut{}, - } - original = []gdbi.Processor{ - &LookupVerts{}, - &HasID{ids: []string{"1", "2", "3"}}, - &Has{stmt: gripql.Within("_gid", "4", "5")}, - &LookupVertexAdjOut{}, - } - optimized = indexStartOptimize(original) + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{V: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_gid", + Value: protoutil.WrapValue([]string{"4", "5"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + } + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_HasId{HasId: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_gid", + Value: protoutil.WrapValue([]string{"4", "5"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + } + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVertsIndex{labels: []string{"foo", "bar"}}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &HasLabel{labels: []string{"foo", "bar"}}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_HasLabel{HasLabel: protoutil.AsListValue([]string{"foo", "bar"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) @@ -146,69 +224,141 @@ func TestIndexStartOptimize(t *testing.T) { } // TODO figure out how to optimize - expected = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Neq("_label", "foo")}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_NEQ, + Key: "_label", + Value: protoutil.WrapValue("foo"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Neq("_label", "foo")}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_NEQ, + Key: "_label", + Value: protoutil.WrapValue("foo"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVertsIndex{labels: []string{"foo", "bar"}}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Within("_label", "foo", "bar")}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_label", + Value: protoutil.WrapValue([]string{"foo", "bar"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVertsIndex{labels: []string{"foo", "bar"}}, - &Has{stmt: gripql.Eq("$.data.foo", "bar")}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Eq("$.data.foo", "bar")}, - &Has{stmt: gripql.Within("_label", "foo", "bar")}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_label", + Value: protoutil.WrapValue([]string{"foo", "bar"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) t.Error("indexStartOptimize returned an unexpected result") } - expected = []gdbi.Processor{ - &LookupVertsIndex{labels: []string{"foo", "bar"}}, - &Has{stmt: gripql.Eq("_label", "baz")}, - &LookupVertexAdjOut{}, + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "_label", + Value: protoutil.WrapValue("baz"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &HasLabel{labels: []string{"foo", "bar"}}, - &Has{stmt: gripql.Eq("_label", "baz")}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_HasLabel{HasLabel: protoutil.AsListValue([]string{"foo", "bar"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "_label", + Value: protoutil.WrapValue("baz"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) @@ -216,20 +366,62 @@ func TestIndexStartOptimize(t *testing.T) { } // use gid over label to optimize queries - expected = []gdbi.Processor{ - &LookupVerts{ids: []string{"1", "2", "3"}}, - &Has{stmt: gripql.Eq("$.data.foo", "bar")}, - &Has{stmt: gripql.Within("_label", "foo", "bar")}, - &LookupVertexAdjOut{}, - } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.Eq("$.data.foo", "bar")}, - &Has{stmt: gripql.Within("_label", "foo", "bar")}, - &Has{stmt: gripql.Within("_gid", "1", "2", "3")}, - &LookupVertexAdjOut{}, - } - optimized = indexStartOptimize(original) + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{V: protoutil.AsListValue([]string{"1", "2", "3"})}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_label", + Value: protoutil.WrapValue([]string{"foo", "bar"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + } + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_label", + Value: protoutil.WrapValue([]string{"foo", "bar"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_gid", + Value: protoutil.WrapValue([]string{"1", "2", "3"}), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, + } + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) @@ -237,17 +429,45 @@ func TestIndexStartOptimize(t *testing.T) { } // handle 'and' statements - expected = []gdbi.Processor{ - &LookupVertsIndex{labels: []string{"foo", "bar"}}, - &Has{stmt: gripql.Eq("$.data.foo", "bar")}, - &LookupVertexAdjOut{}, + + expected = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_LookupVertsIndex{Labels: []string{"foo", "bar"}}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_Condition{ + &gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - original = []gdbi.Processor{ - &LookupVerts{}, - &Has{stmt: gripql.And(gripql.Eq("$.data.foo", "bar"), gripql.Within("_label", "foo", "bar"))}, - &LookupVertexAdjOut{}, + + original = []*gripql.GraphStatement{ + &gripql.GraphStatement{Statement: &gripql.GraphStatement_V{}}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Has{ + Has: &gripql.HasExpression{Expression: &gripql.HasExpression_And{ + &gripql.HasExpressionList{ + []*gripql.HasExpression{ + &gripql.HasExpression{&gripql.HasExpression_Condition{&gripql.HasCondition{ + Condition: gripql.Condition_EQ, + Key: "$.data.foo", + Value: protoutil.WrapValue("bar"), + }}}, + &gripql.HasExpression{&gripql.HasExpression_Condition{&gripql.HasCondition{ + Condition: gripql.Condition_WITHIN, + Key: "_label", + Value: protoutil.WrapValue([]string{"foo", "bar"}), + }}}, + }, + }, + }}, + }}, + &gripql.GraphStatement{Statement: &gripql.GraphStatement_Out{}}, } - optimized = indexStartOptimize(original) + + optimized = IndexStartOptimize(original) if !reflect.DeepEqual(optimized, expected) { t.Log("actual", spew.Sdump(optimized)) t.Log("expected:", spew.Sdump(expected)) diff --git a/engine/inspect/inspect.go b/engine/inspect/inspect.go index 1de2bc90..c2571af1 100644 --- a/engine/inspect/inspect.go +++ b/engine/inspect/inspect.go @@ -1,4 +1,4 @@ -package engine +package inspect import ( "fmt" diff --git a/engine/inspect/inspect_test.go b/engine/inspect/inspect_test.go index 10249cbd..b0604302 100644 --- a/engine/inspect/inspect_test.go +++ b/engine/inspect/inspect_test.go @@ -1,4 +1,4 @@ -package engine +package inspect import ( "fmt" diff --git a/grids/compiler.go b/grids/compiler.go index 0ecd230d..77d8388b 100644 --- a/grids/compiler.go +++ b/grids/compiler.go @@ -1,9 +1,11 @@ package grids import ( + "fmt" "github.com/bmeg/grip/gdbi" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/engine/core" + "github.com/bmeg/grip/engine/inspect" ) @@ -21,6 +23,36 @@ func NewCompiler(ggraph *GridsGraph) gdbi.Compiler { } func (comp GridsCompiler) Compile(stmts []*gripql.GraphStatement) (gdbi.Pipeline, error) { - c := core.NewCompiler(comp.graph) - return c.Compile(stmts) + if len(stmts) == 0 { + return &core.DefaultPipeline{}, nil + } + + stmts = core.Flatten(stmts) + + stmts = core.IndexStartOptimize(stmts) + + if err := core.Validate(stmts); err != nil { + return &core.DefaultPipeline{}, fmt.Errorf("invalid statments: %s", err) + } + + ps := core.NewPipelineState(stmts) + + noLoadPaths := inspect.PipelineNoLoadPathSteps(stmts) + if len(noLoadPaths) > 0 { + fmt.Printf("Found Path: %s\n", noLoadPaths) + } + + procs := make([]gdbi.Processor, 0, len(stmts)) + + for i, gs := range stmts { + ps.SetCurStatment(i) + p, err := core.StatementProcessor(gs, comp.graph, ps) + if err != nil { + return &core.DefaultPipeline{}, err + } + procs = append(procs, p) + } + + + return core.NewPipeline(procs, ps), nil } diff --git a/gripql/custom_statements.go b/gripql/custom_statements.go new file mode 100644 index 00000000..49e39f69 --- /dev/null +++ b/gripql/custom_statements.go @@ -0,0 +1,12 @@ + +package gripql + +//These are custom graph custom statements, which represent operations +//in the traversal that the optimizer may add in, but can't be coded by a +//serialized user request + +type GraphStatement_LookupVertsIndex struct { + Labels []string +} + +func (*GraphStatement_LookupVertsIndex) isGraphStatement_Statement() {}