diff --git a/common/elasticsearch/client_v6.go b/common/elasticsearch/client_v6.go index a6d05649b17..6686ca70cc7 100644 --- a/common/elasticsearch/client_v6.go +++ b/common/elasticsearch/client_v6.go @@ -34,6 +34,7 @@ import ( "github.com/uber/cadence/common" "github.com/uber/cadence/common/config" + "github.com/uber/cadence/common/elasticsearch/query" "github.com/uber/cadence/common/log" "github.com/uber/cadence/common/log/tag" p "github.com/uber/cadence/common/persistence" @@ -53,7 +54,7 @@ type ( // searchParametersV6 holds all required and optional parameters for executing a search searchParametersV6 struct { Index string - Query elastic.Query + Query query.Query From int PageSize int Sorter []elastic.Sorter @@ -126,9 +127,9 @@ func (c *elasticV6) Search(ctx context.Context, request *SearchRequest) (*p.Inte return nil, err } - var matchQuery *elastic.MatchQuery + var matchQuery *query.MatchQuery if request.MatchQuery != nil { - matchQuery = elastic.NewMatchQuery(request.MatchQuery.Name, request.MatchQuery.Text) + matchQuery = query.NewMatchQuery(request.MatchQuery.Name, request.MatchQuery.Text) } searchResult, err := c.getSearchResult( @@ -214,10 +215,10 @@ func (c *elasticV6) SearchForOneClosedExecution( request *p.InternalGetClosedWorkflowExecutionRequest, ) (*p.InternalGetClosedWorkflowExecutionResponse, error) { - matchDomainQuery := elastic.NewMatchQuery(DomainID, request.DomainUUID) - existClosedStatusQuery := elastic.NewExistsQuery(CloseStatus) - matchWorkflowIDQuery := elastic.NewMatchQuery(WorkflowID, request.Execution.GetWorkflowID()) - boolQuery := elastic.NewBoolQuery().Must(matchDomainQuery).Must(existClosedStatusQuery).Must(matchWorkflowIDQuery) + matchDomainQuery := query.NewMatchQuery(DomainID, request.DomainUUID) + existClosedStatusQuery := query.NewExistsQuery(CloseStatus) + matchWorkflowIDQuery := query.NewMatchQuery(WorkflowID, request.Execution.GetWorkflowID()) + boolQuery := query.NewBoolQuery().Must(matchDomainQuery).Must(existClosedStatusQuery).Must(matchWorkflowIDQuery) rid := request.Execution.GetRunID() if rid != "" { matchRunIDQuery := elastic.NewMatchQuery(RunID, rid) @@ -412,13 +413,13 @@ func (c *elasticV6) getSearchResult( ctx context.Context, index string, request *p.InternalListWorkflowExecutionsRequest, - matchQuery *elastic.MatchQuery, + matchQuery *query.MatchQuery, isOpen bool, token *ElasticVisibilityPageToken, ) (*elastic.SearchResult, error) { // always match domain id - boolQuery := elastic.NewBoolQuery().Must(elastic.NewMatchQuery(DomainID, request.DomainUUID)) + boolQuery := query.NewBoolQuery().Must(query.NewMatchQuery(DomainID, request.DomainUUID)) if matchQuery != nil { boolQuery = boolQuery.Must(matchQuery) @@ -434,7 +435,7 @@ func (c *elasticV6) getSearchResult( } var rangeQueryField string - existClosedStatusQuery := elastic.NewExistsQuery(CloseStatus) + existClosedStatusQuery := query.NewExistsQuery(CloseStatus) if isOpen { rangeQueryField = StartTime boolQuery = boolQuery.MustNot(existClosedStatusQuery) @@ -445,7 +446,7 @@ func (c *elasticV6) getSearchResult( earliestTimeStr := strconv.FormatInt(request.EarliestTime.UnixNano()-oneMicroSecondInNano, 10) latestTimeStr := strconv.FormatInt(request.LatestTime.UnixNano()+oneMicroSecondInNano, 10) - rangeQuery := elastic.NewRangeQuery(rangeQueryField).Gte(earliestTimeStr).Lte(latestTimeStr) + rangeQuery := query.NewRangeQuery(rangeQueryField).Gte(earliestTimeStr).Lte(latestTimeStr) boolQuery = boolQuery.Filter(rangeQuery) params := &searchParametersV6{ @@ -455,8 +456,8 @@ func (c *elasticV6) getSearchResult( PageSize: request.PageSize, } - params.Sorter = append(params.Sorter, elastic.NewFieldSort(rangeQueryField).Desc()) - params.Sorter = append(params.Sorter, elastic.NewFieldSort(RunID).Desc()) + params.Sorter = append(params.Sorter, query.NewFieldSort(rangeQueryField).Desc()) + params.Sorter = append(params.Sorter, query.NewFieldSort(RunID).Desc()) if ShouldSearchAfter(token) { params.SearchAfter = []interface{}{token.SortValue, token.TieBreaker} diff --git a/common/elasticsearch/client_v7.go b/common/elasticsearch/client_v7.go index 3e06c4e4d0a..d78095ca590 100644 --- a/common/elasticsearch/client_v7.go +++ b/common/elasticsearch/client_v7.go @@ -34,6 +34,7 @@ import ( "github.com/uber/cadence/common" "github.com/uber/cadence/common/config" + "github.com/uber/cadence/common/elasticsearch/query" "github.com/uber/cadence/common/log" "github.com/uber/cadence/common/log/tag" p "github.com/uber/cadence/common/persistence" @@ -53,7 +54,7 @@ type ( // searchParametersV7 holds all required and optional parameters for executing a search searchParametersV7 struct { Index string - Query elastic.Query + Query query.Query From int PageSize int Sorter []elastic.Sorter @@ -126,9 +127,9 @@ func (c *elasticV7) Search(ctx context.Context, request *SearchRequest) (*p.Inte return nil, err } - var matchQuery *elastic.MatchQuery + var matchQuery *query.MatchQuery if request.MatchQuery != nil { - matchQuery = elastic.NewMatchQuery(request.MatchQuery.Name, request.MatchQuery.Text) + matchQuery = query.NewMatchQuery(request.MatchQuery.Name, request.MatchQuery.Text) } searchResult, err := c.getSearchResult( @@ -214,13 +215,13 @@ func (c *elasticV7) SearchForOneClosedExecution( request *p.InternalGetClosedWorkflowExecutionRequest, ) (*p.InternalGetClosedWorkflowExecutionResponse, error) { - matchDomainQuery := elastic.NewMatchQuery(DomainID, request.DomainUUID) - existClosedStatusQuery := elastic.NewExistsQuery(CloseStatus) - matchWorkflowIDQuery := elastic.NewMatchQuery(WorkflowID, request.Execution.GetWorkflowID()) - boolQuery := elastic.NewBoolQuery().Must(matchDomainQuery).Must(existClosedStatusQuery).Must(matchWorkflowIDQuery) + matchDomainQuery := query.NewMatchQuery(DomainID, request.DomainUUID) + existClosedStatusQuery := query.NewExistsQuery(CloseStatus) + matchWorkflowIDQuery := query.NewMatchQuery(WorkflowID, request.Execution.GetWorkflowID()) + boolQuery := query.NewBoolQuery().Must(matchDomainQuery).Must(existClosedStatusQuery).Must(matchWorkflowIDQuery) rid := request.Execution.GetRunID() if rid != "" { - matchRunIDQuery := elastic.NewMatchQuery(RunID, rid) + matchRunIDQuery := query.NewMatchQuery(RunID, rid) boolQuery = boolQuery.Must(matchRunIDQuery) } @@ -415,14 +416,13 @@ func (c *elasticV7) getSearchResult( ctx context.Context, index string, request *p.InternalListWorkflowExecutionsRequest, - matchQuery *elastic.MatchQuery, + matchQuery *query.MatchQuery, isOpen bool, token *ElasticVisibilityPageToken, ) (*elastic.SearchResult, error) { // always match domain id - boolQuery := elastic.NewBoolQuery().Must(elastic.NewMatchQuery(DomainID, request.DomainUUID)) - + boolQuery := query.NewBoolQuery().Must(query.NewMatchQuery(DomainID, request.DomainUUID)) if matchQuery != nil { boolQuery = boolQuery.Must(matchQuery) } @@ -457,9 +457,8 @@ func (c *elasticV7) getSearchResult( From: token.From, PageSize: request.PageSize, } - - params.Sorter = append(params.Sorter, elastic.NewFieldSort(rangeQueryField).Desc()) - params.Sorter = append(params.Sorter, elastic.NewFieldSort(RunID).Desc()) + params.Sorter = append(params.Sorter, query.NewFieldSort(rangeQueryField).Desc()) + params.Sorter = append(params.Sorter, query.NewFieldSort(RunID).Desc()) if ShouldSearchAfter(token) { params.SearchAfter = []interface{}{token.SortValue, token.TieBreaker} diff --git a/common/elasticsearch/query/bool_query.go b/common/elasticsearch/query/bool_query.go new file mode 100644 index 00000000000..ce199ed730f --- /dev/null +++ b/common/elasticsearch/query/bool_query.go @@ -0,0 +1,124 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +type Query interface { + // Source returns the JSON-serializable query request. + Source() (interface{}, error) +} + +type BoolQuery struct { + mustClauses []Query + mustNotClauses []Query + filterClauses []Query +} + +// Creates a new bool query. +func NewBoolQuery() *BoolQuery { + return &BoolQuery{ + mustClauses: make([]Query, 0), + mustNotClauses: make([]Query, 0), + filterClauses: make([]Query, 0), + } +} + +func (q *BoolQuery) Must(queries ...Query) *BoolQuery { + q.mustClauses = append(q.mustClauses, queries...) + return q +} + +func (q *BoolQuery) MustNot(queries ...Query) *BoolQuery { + q.mustNotClauses = append(q.mustNotClauses, queries...) + return q +} + +func (q *BoolQuery) Filter(filters ...Query) *BoolQuery { + q.filterClauses = append(q.filterClauses, filters...) + return q +} + +func (q *BoolQuery) Source() (interface{}, error) { + query := make(map[string]interface{}) + + boolClause := make(map[string]interface{}) + query["bool"] = boolClause + + // must + if len(q.mustClauses) == 1 { + src, err := q.mustClauses[0].Source() + if err != nil { + return nil, err + } + boolClause["must"] = src + } else if len(q.mustClauses) > 1 { + var clauses []interface{} + for _, subQuery := range q.mustClauses { + src, err := subQuery.Source() + if err != nil { + return nil, err + } + clauses = append(clauses, src) + } + boolClause["must"] = clauses + } + + // must_not + if len(q.mustNotClauses) == 1 { + src, err := q.mustNotClauses[0].Source() + if err != nil { + return nil, err + } + boolClause["must_not"] = src + } else if len(q.mustNotClauses) > 1 { + var clauses []interface{} + for _, subQuery := range q.mustNotClauses { + src, err := subQuery.Source() + if err != nil { + return nil, err + } + clauses = append(clauses, src) + } + boolClause["must_not"] = clauses + } + + // filter + if len(q.filterClauses) == 1 { + src, err := q.filterClauses[0].Source() + if err != nil { + return nil, err + } + boolClause["filter"] = src + } else if len(q.filterClauses) > 1 { + var clauses []interface{} + for _, subQuery := range q.filterClauses { + src, err := subQuery.Source() + if err != nil { + return nil, err + } + clauses = append(clauses, src) + } + boolClause["filter"] = clauses + } + + return query, nil +} diff --git a/common/elasticsearch/query/bool_query_test.go b/common/elasticsearch/query/bool_query_test.go new file mode 100644 index 00000000000..c6ce80ea40d --- /dev/null +++ b/common/elasticsearch/query/bool_query_test.go @@ -0,0 +1,46 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +import ( + "encoding/json" + "testing" +) + +func TestBoolQuery(t *testing.T) { + q := NewBoolQuery() + q = q.MustNot(NewRangeQuery("age").From(10).To(20)) + src, err := q.Source() + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"bool":{"must_not":{"range":{"age":{"from":10,"include_lower":true,"include_upper":true,"to":20}}}}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} diff --git a/common/elasticsearch/query/builder.go b/common/elasticsearch/query/builder.go new file mode 100644 index 00000000000..ff6578b892f --- /dev/null +++ b/common/elasticsearch/query/builder.go @@ -0,0 +1,117 @@ +package query + +import "encoding/json" + +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +type Builder struct { + query Query // query + from int // from + size int // size + sorters []Sorter // sort + searchAfterSortValues []interface{} // search_after + +} + +func NewBuilder() *Builder { + return &Builder{ + from: -1, + size: -1, + } +} + +func (b *Builder) Query(query Query) *Builder { + b.query = query + return b +} + +func (b *Builder) From(from int) *Builder { + b.from = from + return b +} + +func (b *Builder) Sortby(sorters ...Sorter) *Builder { + b.sorters = sorters + return b +} + +func (b *Builder) Size(size int) *Builder { + b.size = size + return b +} + +func (b *Builder) SearchAfter(v ...interface{}) *Builder { + b.searchAfterSortValues = v + return b +} + +// Source returns the serializable JSON for the source builder. +func (b *Builder) Source() (interface{}, error) { + source := make(map[string]interface{}) + + if b.from != -1 { + source["from"] = b.from + } + if b.size != -1 { + source["size"] = b.size + } + + if b.query != nil { + src, err := b.query.Source() + if err != nil { + return nil, err + } + source["query"] = src + } + if len(b.sorters) > 0 { + var sortarr []interface{} + for _, sorter := range b.sorters { + src, err := sorter.Source() + if err != nil { + return nil, err + } + sortarr = append(sortarr, src) + } + source["sort"] = sortarr + } + + if len(b.searchAfterSortValues) > 0 { + source["search_after"] = b.searchAfterSortValues + } + + return source, nil +} + +func (b *Builder) String() (string, error) { + source, err := b.Source() + if err != nil { + return "", err + } + + marshaled, err := json.Marshal(source) + if err != nil { + return "", err + } + + return string(marshaled), nil +} diff --git a/common/elasticsearch/query/builder_test.go b/common/elasticsearch/query/builder_test.go new file mode 100644 index 00000000000..41405967de4 --- /dev/null +++ b/common/elasticsearch/query/builder_test.go @@ -0,0 +1,74 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +import ( + "encoding/json" + + "github.com/olivere/elastic/v7" + "github.com/stretchr/testify/assert" + + "testing" +) + +func TestQueryBuilder(t *testing.T) { + qb := NewBuilder() + qb.Query(NewExistsQuery("user")) + qb.Size(10) + qb.From(100) + qb.Sortby(NewFieldSort("StartDate")) + src, err := qb.Source() + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"from":100,"query":{"exists":{"field":"user"}},"size":10,"sort":[{"StartDate":{"order":"asc"}}]}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} + +func TestBuilderAgainsESv7(t *testing.T) { + qb := NewBuilder() + qb.Query(NewExistsQuery("user")) + qb.Size(10) + qb.Sortby(NewFieldSort("runid").Desc()) + qb.Query(NewBoolQuery().Must(NewMatchQuery("domainID", "uuid"))).SearchAfter([]interface{}{"sortval", "tiebraker"}) + qbs, err := qb.Source() + assert.NoError(t, err) + + searchSource := elastic.NewSearchSource(). + Query(elastic.NewExistsQuery("user")). + Size(10). + SortBy(elastic.NewFieldSort("runid").Desc()). + Query(elastic.NewBoolQuery().Must(elastic.NewMatchQuery("domainID", "uuid"))).SearchAfter([]interface{}{"sortval", "tiebraker"}) + + sss, err := searchSource.Source() + assert.NoError(t, err) + + assert.Equal(t, sss, qbs, "ESv7 and local QueryBuilder should produce the same query") +} diff --git a/common/elasticsearch/query/exists_query.go b/common/elasticsearch/query/exists_query.go new file mode 100644 index 00000000000..502bde12756 --- /dev/null +++ b/common/elasticsearch/query/exists_query.go @@ -0,0 +1,67 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +// ExistsQuery is a query that only matches on documents that the field +// has a value in them. +// +// For more details, see: +// https://www.elastic.co/guide/en/elasticsearch/reference/6.8/query-dsl-exists-query.html +type ExistsQuery struct { + name string + queryName string +} + +// NewExistsQuery creates and initializes a new exists query. +func NewExistsQuery(name string) *ExistsQuery { + return &ExistsQuery{ + name: name, + } +} + +// QueryName sets the query name for the filter that can be used +// when searching for matched queries per hit. +func (q *ExistsQuery) QueryName(queryName string) *ExistsQuery { + q.queryName = queryName + return q +} + +// Source returns the JSON serializable content for this query. +func (q *ExistsQuery) Source() (interface{}, error) { + // { + // "exists" : { + // "field" : "user" + // } + // } + + query := make(map[string]interface{}) + params := make(map[string]interface{}) + query["exists"] = params + + params["field"] = q.name + if q.queryName != "" { + params["_name"] = q.queryName + } + + return query, nil +} diff --git a/common/elasticsearch/query/exists_query_test.go b/common/elasticsearch/query/exists_query_test.go new file mode 100644 index 00000000000..dcf2b1e4ed3 --- /dev/null +++ b/common/elasticsearch/query/exists_query_test.go @@ -0,0 +1,45 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +import ( + "encoding/json" + "testing" +) + +func TestExistsQuery(t *testing.T) { + q := NewExistsQuery("user") + src, err := q.Source() + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"exists":{"field":"user"}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} diff --git a/common/elasticsearch/query/match_query.go b/common/elasticsearch/query/match_query.go new file mode 100644 index 00000000000..072634350f0 --- /dev/null +++ b/common/elasticsearch/query/match_query.go @@ -0,0 +1,207 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +// MatchQuery is a family of queries that accepts text/numerics/dates, +// analyzes them, and constructs a query. +// +// To create a new MatchQuery, use NewMatchQuery. To create specific types +// of queries, e.g. a match_phrase query, use NewMatchPhrQuery(...).Type("phrase"), +// or use one of the shortcuts e.g. NewMatchPhraseQuery(...). +// +// For more details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-match-query.html +type MatchQuery struct { + name string + text interface{} + operator string // or / and + analyzer string + boost *float64 + fuzziness string + prefixLength *int + maxExpansions *int + minimumShouldMatch string + fuzzyRewrite string + lenient *bool + fuzzyTranspositions *bool + zeroTermsQuery string + cutoffFrequency *float64 + queryName string +} + +// NewMatchQuery creates and initializes a new MatchQuery. +func NewMatchQuery(name string, text interface{}) *MatchQuery { + return &MatchQuery{name: name, text: text} +} + +// Operator sets the operator to use when using a boolean query. +// Can be "AND" or "OR" (default). +func (q *MatchQuery) Operator(operator string) *MatchQuery { + q.operator = operator + return q +} + +// Analyzer explicitly sets the analyzer to use. It defaults to use explicit +// mapping config for the field, or, if not set, the default search analyzer. +func (q *MatchQuery) Analyzer(analyzer string) *MatchQuery { + q.analyzer = analyzer + return q +} + +// Fuzziness sets the fuzziness when evaluated to a fuzzy query type. +// Defaults to "AUTO". +func (q *MatchQuery) Fuzziness(fuzziness string) *MatchQuery { + q.fuzziness = fuzziness + return q +} + +// PrefixLength sets the length of a length of common (non-fuzzy) +// prefix for fuzzy match queries. It must be non-negative. +func (q *MatchQuery) PrefixLength(prefixLength int) *MatchQuery { + q.prefixLength = &prefixLength + return q +} + +// MaxExpansions is used with fuzzy or prefix type queries. It specifies +// the number of term expansions to use. It defaults to unbounded so that +// its recommended to set it to a reasonable value for faster execution. +func (q *MatchQuery) MaxExpansions(maxExpansions int) *MatchQuery { + q.maxExpansions = &maxExpansions + return q +} + +// CutoffFrequency can be a value in [0..1] (or an absolute number >=1). +// It represents the maximum treshold of a terms document frequency to be +// considered a low frequency term. +func (q *MatchQuery) CutoffFrequency(cutoff float64) *MatchQuery { + q.cutoffFrequency = &cutoff + return q +} + +// MinimumShouldMatch sets the optional minimumShouldMatch value to +// apply to the query. +func (q *MatchQuery) MinimumShouldMatch(minimumShouldMatch string) *MatchQuery { + q.minimumShouldMatch = minimumShouldMatch + return q +} + +// FuzzyRewrite sets the fuzzy_rewrite parameter controlling how the +// fuzzy query will get rewritten. +func (q *MatchQuery) FuzzyRewrite(fuzzyRewrite string) *MatchQuery { + q.fuzzyRewrite = fuzzyRewrite + return q +} + +// FuzzyTranspositions sets whether transpositions are supported in +// fuzzy queries. +// +// The default metric used by fuzzy queries to determine a match is +// the Damerau-Levenshtein distance formula which supports transpositions. +// Setting transposition to false will +// * switch to classic Levenshtein distance. +// * If not set, Damerau-Levenshtein distance metric will be used. +func (q *MatchQuery) FuzzyTranspositions(fuzzyTranspositions bool) *MatchQuery { + q.fuzzyTranspositions = &fuzzyTranspositions + return q +} + +// Lenient specifies whether format based failures will be ignored. +func (q *MatchQuery) Lenient(lenient bool) *MatchQuery { + q.lenient = &lenient + return q +} + +// ZeroTermsQuery can be "all" or "none". +func (q *MatchQuery) ZeroTermsQuery(zeroTermsQuery string) *MatchQuery { + q.zeroTermsQuery = zeroTermsQuery + return q +} + +// Boost sets the boost to apply to this query. +func (q *MatchQuery) Boost(boost float64) *MatchQuery { + q.boost = &boost + return q +} + +// QueryName sets the query name for the filter that can be used when +// searching for matched filters per hit. +func (q *MatchQuery) QueryName(queryName string) *MatchQuery { + q.queryName = queryName + return q +} + +// Source returns JSON for the function score query. +func (q *MatchQuery) Source() (interface{}, error) { + // {"match":{"name":{"query":"value","type":"boolean/phrase"}}} + source := make(map[string]interface{}) + + match := make(map[string]interface{}) + source["match"] = match + + query := make(map[string]interface{}) + match[q.name] = query + + query["query"] = q.text + + if q.operator != "" { + query["operator"] = q.operator + } + if q.analyzer != "" { + query["analyzer"] = q.analyzer + } + if q.fuzziness != "" { + query["fuzziness"] = q.fuzziness + } + if q.prefixLength != nil { + query["prefix_length"] = *q.prefixLength + } + if q.maxExpansions != nil { + query["max_expansions"] = *q.maxExpansions + } + if q.minimumShouldMatch != "" { + query["minimum_should_match"] = q.minimumShouldMatch + } + if q.fuzzyRewrite != "" { + query["fuzzy_rewrite"] = q.fuzzyRewrite + } + if q.lenient != nil { + query["lenient"] = *q.lenient + } + if q.fuzzyTranspositions != nil { + query["fuzzy_transpositions"] = *q.fuzzyTranspositions + } + if q.zeroTermsQuery != "" { + query["zero_terms_query"] = q.zeroTermsQuery + } + if q.cutoffFrequency != nil { + query["cutoff_frequency"] = *q.cutoffFrequency + } + if q.boost != nil { + query["boost"] = *q.boost + } + if q.queryName != "" { + query["_name"] = q.queryName + } + + return source, nil +} diff --git a/common/elasticsearch/query/match_query_test.go b/common/elasticsearch/query/match_query_test.go new file mode 100644 index 00000000000..d5cc62aa7ce --- /dev/null +++ b/common/elasticsearch/query/match_query_test.go @@ -0,0 +1,62 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +import ( + "encoding/json" + "testing" +) + +func TestMatchQuery(t *testing.T) { + q := NewMatchQuery("message", "this is a test") + src, err := q.Source() + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"match":{"message":{"query":"this is a test"}}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} + +func TestMatchQueryWithOptions(t *testing.T) { + q := NewMatchQuery("message", "this is a test").Analyzer("whitespace").Operator("or").Boost(2.5) + src, err := q.Source() + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"match":{"message":{"analyzer":"whitespace","boost":2.5,"operator":"or","query":"this is a test"}}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} diff --git a/common/elasticsearch/query/range_query.go b/common/elasticsearch/query/range_query.go new file mode 100644 index 00000000000..5a57aaa6aaa --- /dev/null +++ b/common/elasticsearch/query/range_query.go @@ -0,0 +1,118 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +// RangeQuery matches documents with fields that have terms within a certain range. +// +// For details, see +// https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-range-query.html +type RangeQuery struct { + name string + from interface{} + to interface{} + includeLower bool + includeUpper bool +} + +// NewRangeQuery creates and initializes a new RangeQuery. +func NewRangeQuery(name string) *RangeQuery { + return &RangeQuery{name: name, includeLower: true, includeUpper: true} +} + +// From indicates the from part of the RangeQuery. +// Use nil to indicate an unbounded from part. +func (q *RangeQuery) From(from interface{}) *RangeQuery { + q.from = from + return q +} + +// Gt indicates a greater-than value for the from part. +// Use nil to indicate an unbounded from part. +func (q *RangeQuery) Gt(from interface{}) *RangeQuery { + q.from = from + q.includeLower = false + return q +} + +// Gte indicates a greater-than-or-equal value for the from part. +// Use nil to indicate an unbounded from part. +func (q *RangeQuery) Gte(from interface{}) *RangeQuery { + q.from = from + q.includeLower = true + return q +} + +// To indicates the to part of the RangeQuery. +// Use nil to indicate an unbounded to part. +func (q *RangeQuery) To(to interface{}) *RangeQuery { + q.to = to + return q +} + +// Lt indicates a less-than value for the to part. +// Use nil to indicate an unbounded to part. +func (q *RangeQuery) Lt(to interface{}) *RangeQuery { + q.to = to + q.includeUpper = false + return q +} + +// Lte indicates a less-than-or-equal value for the to part. +// Use nil to indicate an unbounded to part. +func (q *RangeQuery) Lte(to interface{}) *RangeQuery { + q.to = to + q.includeUpper = true + return q +} + +// IncludeLower indicates whether the lower bound should be included or not. +// Defaults to true. +func (q *RangeQuery) IncludeLower(includeLower bool) *RangeQuery { + q.includeLower = includeLower + return q +} + +// IncludeUpper indicates whether the upper bound should be included or not. +// Defaults to true. +func (q *RangeQuery) IncludeUpper(includeUpper bool) *RangeQuery { + q.includeUpper = includeUpper + return q +} + +// Source returns JSON for the query. +func (q *RangeQuery) Source() (interface{}, error) { + source := make(map[string]interface{}) + + rangeQ := make(map[string]interface{}) + source["range"] = rangeQ + + params := make(map[string]interface{}) + rangeQ[q.name] = params + + params["from"] = q.from + params["to"] = q.to + params["include_lower"] = q.includeLower + params["include_upper"] = q.includeUpper + + return source, nil +} diff --git a/common/elasticsearch/query/range_query_test.go b/common/elasticsearch/query/range_query_test.go new file mode 100644 index 00000000000..c7667045436 --- /dev/null +++ b/common/elasticsearch/query/range_query_test.go @@ -0,0 +1,45 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +import ( + "encoding/json" + "testing" +) + +func TestRangeQuery(t *testing.T) { + q := NewRangeQuery("postDate").From("2010-03-01").To("2010-04-01") + src, err := q.Source() + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"range":{"postDate":{"from":"2010-03-01","include_lower":true,"include_upper":true,"to":"2010-04-01"}}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +} diff --git a/common/elasticsearch/query/sort.go b/common/elasticsearch/query/sort.go new file mode 100644 index 00000000000..9b39165b6b9 --- /dev/null +++ b/common/elasticsearch/query/sort.go @@ -0,0 +1,80 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +type Sorter interface { + Source() (interface{}, error) +} + +// FieldSort sorts by a given field. +type FieldSort struct { + Sorter + fieldName string + ascending bool +} + +// NewFieldSort creates a new FieldSort. +func NewFieldSort(fieldName string) *FieldSort { + return &FieldSort{ + fieldName: fieldName, + ascending: true, + } +} + +// FieldName specifies the name of the field to be used for sorting. +func (s *FieldSort) FieldName(fieldName string) *FieldSort { + s.fieldName = fieldName + return s +} + +// Order defines whether sorting ascending (default) or descending. +func (s *FieldSort) Order(ascending bool) *FieldSort { + s.ascending = ascending + return s +} + +// Asc sets ascending sort order. +func (s *FieldSort) Asc() *FieldSort { + s.ascending = true + return s +} + +// Desc sets descending sort order. +func (s *FieldSort) Desc() *FieldSort { + s.ascending = false + return s +} + +// Source returns the JSON-serializable data. +func (s *FieldSort) Source() (interface{}, error) { + source := make(map[string]interface{}) + x := make(map[string]interface{}) + source[s.fieldName] = x + if s.ascending { + x["order"] = "asc" + } else { + x["order"] = "desc" + } + + return source, nil +} diff --git a/common/elasticsearch/query/sort_test.go b/common/elasticsearch/query/sort_test.go new file mode 100644 index 00000000000..18d3238db46 --- /dev/null +++ b/common/elasticsearch/query/sort_test.go @@ -0,0 +1,45 @@ +// The MIT License (MIT) + +// Copyright (c) 2017-2020 Uber Technologies Inc. + +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package query + +import ( + "encoding/json" + "testing" +) + +func TestFieldSort(t *testing.T) { + builder := NewFieldSort("grade") + src, err := builder.Source() + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(src) + if err != nil { + t.Fatalf("marshaling to JSON failed: %v", err) + } + got := string(data) + expected := `{"grade":{"order":"asc"}}` + if got != expected { + t.Errorf("expected\n%s\n,got:\n%s", expected, got) + } +}