Skip to content

Commit

Permalink
Allow for multiple MATCH patterns
Browse files Browse the repository at this point in the history 
Each MATCH is inclusive OR, thus

    WITHIN fleet MATCH train* truck* BOUNDS 33 -112 34 -113

will find all trains and trucks that within the provides bounds.
  • Loading branch information
@tidwall
tidwall committed Sep 2, 2022
1 parent d295330 commit f24c251
Show file tree
Hide file tree
Showing 8 changed files with 96 additions and 61 deletions.
21 changes: 16 additions & 5 deletions internal/server/fence.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,20 @@ func objIsSpatial(obj geojson.Object) bool {
func hookJSONString(hookName string, metas []FenceMeta) string {
return string(appendHookDetails(nil, hookName, metas))
}

func multiGlobMatch(globs []string, s string) bool {
if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") {
return true
}
for _, pattern := range globs {
match, _ := glob.Match(pattern, s)
if match {
return true
}
}
return false
}

func fenceMatch(
hookName string, sw *scanWriter, fence *liveFenceSwitches,
metas []FenceMeta, details *commandDetails,
Expand All @@ -66,11 +80,8 @@ func fenceMatch(
`,"time":` + jsonTimeFormat(details.timestamp) + `}`,
}
}
if len(fence.glob) > 0 && !(len(fence.glob) == 1 && fence.glob[0] == '*') {
match, _ := glob.Match(fence.glob, details.id)
if !match {
return nil
}
if !multiGlobMatch(fence.globs, details.id) {
return nil
}
if details.obj == nil || !objIsSpatial(details.obj) {
return nil
Expand Down
2 changes: 1 addition & 1 deletion internal/server/hooks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ func (s *Server) cmdSetHook(msg *Message) (
}
var wr bytes.Buffer
hook.ScanWriter, err = s.newScanWriter(
&wr, cmsg, args.key, args.output, args.precision, args.glob, false,
&wr, cmsg, args.key, args.output, args.precision, args.globs, false,
args.cursor, args.limit, args.wheres, args.whereins, args.whereevals,
args.nofields)
if err != nil {
Expand Down
6 changes: 3 additions & 3 deletions internal/server/live.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (

type liveBuffer struct {
key string
glob string
globs []string
fence *liveFenceSwitches
details []*commandDetails
cond *sync.Cond
Expand Down Expand Up @@ -101,12 +101,12 @@ func (s *Server) goLive(
var sw *scanWriter
var wr bytes.Buffer
lfs := inerr.(liveFenceSwitches)
lb.glob = lfs.glob
lb.globs = lfs.globs
lb.key = lfs.key
lb.fence = &lfs
s.mu.RLock()
sw, err = s.newScanWriter(
&wr, msg, lfs.key, lfs.output, lfs.precision, lfs.glob, false,
&wr, msg, lfs.key, lfs.output, lfs.precision, lfs.globs, false,
lfs.cursor, lfs.limit, lfs.wheres, lfs.whereins, lfs.whereevals, lfs.nofields)
s.mu.RUnlock()

Expand Down
9 changes: 4 additions & 5 deletions internal/server/scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (

"github.com/tidwall/geojson"
"github.com/tidwall/resp"
"github.com/tidwall/tile38/internal/glob"
)

func (s *Server) cmdScanArgs(vs []string) (
Expand Down Expand Up @@ -46,7 +45,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
}
wr := &bytes.Buffer{}
sw, err := s.newScanWriter(
wr, msg, args.key, args.output, args.precision, args.glob, false,
wr, msg, args.key, args.output, args.precision, args.globs, false,
args.cursor, args.limit, args.wheres, args.whereins, args.whereevals,
args.nofields)
if err != nil {
Expand All @@ -65,8 +64,8 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
}
sw.count = uint64(count)
} else {
g := glob.Parse(sw.globPattern, args.desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
limits := multiGlobParse(sw.globs, args.desc)
if limits[0] == "" && limits[1] == "" {
sw.col.Scan(args.desc, sw,
msg.Deadline,
func(id string, o geojson.Object, fields []float64) bool {
Expand All @@ -78,7 +77,7 @@ func (s *Server) cmdScan(msg *Message) (res resp.Value, err error) {
},
)
} else {
sw.col.ScanRange(g.Limits[0], g.Limits[1], args.desc, sw,
sw.col.ScanRange(limits[0], limits[1], args.desc, sw,
msg.Deadline,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(ScanWriterParams{
Expand Down
46 changes: 20 additions & 26 deletions internal/server/scanner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,8 @@ type scanWriter struct {
once bool
count uint64
precision uint64
globPattern string
globs []string
globEverything bool
globSingle bool
fullFields bool
values []resp.Value
matchValues bool
Expand All @@ -79,7 +78,7 @@ type ScanWriterParams struct {

func (s *Server) newScanWriter(
wr *bytes.Buffer, msg *Message, key string, output outputT,
precision uint64, globPattern string, matchValues bool,
precision uint64, globs []string, matchValues bool,
cursor, limit uint64, wheres []whereT, whereins []whereinT,
whereevals []whereevalT, nofields bool,
) (
Expand All @@ -102,21 +101,18 @@ func (s *Server) newScanWriter(
wr: wr,
key: key,
msg: msg,
globs: globs,
limit: limit,
cursor: cursor,
output: output,
nofields: nofields,
precision: precision,
whereevals: whereevals,
globPattern: globPattern,
matchValues: matchValues,
}
if globPattern == "*" || globPattern == "" {

if len(globs) == 0 || (len(globs) == 1 && globs[0] == "*") {
sw.globEverything = true
} else {
if !glob.IsGlob(globPattern) {
sw.globSingle = true
}
}
sw.orgWheres = wheres
sw.orgWhereins = whereins
Expand Down Expand Up @@ -344,25 +340,23 @@ func (sw *scanWriter) fieldMatch(fields []float64, o geojson.Object) (fvals []fl
}

func (sw *scanWriter) globMatch(id string, o geojson.Object) (ok, keepGoing bool) {
if !sw.globEverything {
if sw.globSingle {
if sw.globPattern != id {
return false, true
}
return true, false
}
var val string
if sw.matchValues {
val = o.String()
} else {
val = id
}
ok, _ := glob.Match(sw.globPattern, val)
if !ok {
return false, true
if sw.globEverything {
return true, true
}
var val string
if sw.matchValues {
val = o.String()
} else {
val = id
}
for _, pattern := range sw.globs {
ok, _ := glob.Match(pattern, val)
if ok {
return true, true
}
}
return true, true
return false, true

}

// Increment cursor
Expand Down
42 changes: 35 additions & 7 deletions internal/server/search.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ func (s *Server) cmdNearby(msg *Message) (res resp.Value, err error) {
return NOMessage, sargs
}
sw, err := s.newScanWriter(
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false,
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false,
sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields)
if err != nil {
return NOMessage, err
Expand Down Expand Up @@ -580,7 +580,7 @@ func (s *Server) cmdWithinOrIntersects(cmd string, msg *Message) (res resp.Value
return NOMessage, sargs
}
sw, err := s.newScanWriter(
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, false,
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, false,
sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields)
if err != nil {
return NOMessage, err
Expand Down Expand Up @@ -644,6 +644,35 @@ func (s *Server) cmdSeachValuesArgs(vs []string) (
return
}

func multiGlobParse(globs []string, desc bool) [2]string {
var limits [2]string
for i, pattern := range globs {
g := glob.Parse(pattern, desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
limits[0], limits[1] = "", ""
break
}
if i == 0 {
limits[0], limits[1] = g.Limits[0], g.Limits[1]
} else if desc {
if g.Limits[0] > limits[0] {
limits[0] = g.Limits[0]
}
if g.Limits[1] < limits[1] {
limits[1] = g.Limits[1]
}
} else {
if g.Limits[0] < limits[0] {
limits[0] = g.Limits[0]
}
if g.Limits[1] > limits[1] {
limits[1] = g.Limits[1]
}
}
}
return limits
}

func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
start := time.Now()
vs := msg.Args[1:]
Expand All @@ -664,7 +693,7 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
return NOMessage, err
}
sw, err := s.newScanWriter(
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.glob, true,
wr, msg, sargs.key, sargs.output, sargs.precision, sargs.globs, true,
sargs.cursor, sargs.limit, sargs.wheres, sargs.whereins, sargs.whereevals, sargs.nofields)
if err != nil {
return NOMessage, err
Expand All @@ -681,8 +710,8 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
}
sw.count = uint64(count)
} else {
g := glob.Parse(sw.globPattern, sargs.desc)
if g.Limits[0] == "" && g.Limits[1] == "" {
limits := multiGlobParse(sw.globs, sargs.desc)
if limits[0] == "" && limits[1] == "" {
sw.col.SearchValues(sargs.desc, sw, msg.Deadline,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(ScanWriterParams{
Expand All @@ -696,8 +725,7 @@ func (s *Server) cmdSearch(msg *Message) (res resp.Value, err error) {
} else {
// must disable globSingle for string value type matching because
// globSingle is only for ID matches, not values.
sw.globSingle = false
sw.col.SearchValuesRange(g.Limits[0], g.Limits[1], sargs.desc, sw,
sw.col.SearchValuesRange(limits[0], limits[1], sargs.desc, sw,
msg.Deadline,
func(id string, o geojson.Object, fields []float64) bool {
return sw.writeObject(ScanWriterParams{
Expand Down
10 changes: 4 additions & 6 deletions internal/server/token.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ type searchScanBaseTokens struct {
nodwell bool
detect map[string]bool
accept map[string]bool
glob string
globs []string
wheres []whereT
whereins []whereinT
whereevals []whereevalT
Expand Down Expand Up @@ -543,14 +543,12 @@ func (s *Server) parseSearchScanBaseTokens(
continue
case "match":
vs = nvs
if t.glob != "" {
err = errDuplicateArgument(strings.ToUpper(wtok))
return
}
if vs, t.glob, ok = tokenval(vs); !ok || t.glob == "" {
var glob string
if vs, glob, ok = tokenval(vs); !ok || glob == "" {
err = errInvalidNumberOfArguments
return
}
t.globs = append(t.globs, glob)
continue
case "clip":
vs = nvs
Expand Down
21 changes: 13 additions & 8 deletions tests/keys_search_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -591,23 +591,28 @@ func keys_MATCH_test(mc *mockServer) error {
{"SET", "fleet", "truck2", "POINT", "33.0002", "-112.0002"}, {"OK"},
{"SET", "fleet", "luck1", "POINT", "33.0003", "-112.0003"}, {"OK"},
{"SET", "fleet", "luck2", "POINT", "33.0004", "-112.0004"}, {"OK"},
{"SET", "fleet", "train1", "POINT", "33.0005", "-112.0005"}, {"OK"},

{"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"},
{"SCAN", "fleet", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "truck*", "IDS"}, {"[0 [truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "luck*", "IDS"}, {"[0 [luck1 luck2]]"},
{"SCAN", "fleet", "MATCH", "*2", "IDS"}, {"[0 [luck2 truck2]]"},
{"SCAN", "fleet", "MATCH", "*2*", "IDS"}, {"[0 [luck2 truck2]]"},
{"SCAN", "fleet", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*u*", "IDS"}, {"[0 [luck1 luck2 truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "*u*", "MATCH", "*a*", "IDS"}, {"[0 [luck1 luck2 train1 truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "IDS"}, {"[0 [train1 truck1 truck2]]"},
{"SCAN", "fleet", "MATCH", "train*", "MATCH", "truck*", "MATCH", "luck1", "IDS"}, {"[0 [luck1 train1 truck1 truck2]]"},

{"NEARBY", "fleet", "IDS", "POINT", 33.00005, -112.00005, 100000}, {
match("[0 [luck1 luck2 truck1 truck2]]"),
match("[0 [luck1 luck2 train1 truck1 truck2]]"),
},
{"NEARBY", "fleet", "MATCH", "*", "IDS", "POINT", 33.00005, -112.00005, 100000}, {
match("[0 [luck1 luck2 truck1 truck2]]"),
match("[0 [luck1 luck2 train1 truck1 truck2]]"),
},
{"NEARBY", "fleet", "MATCH", "t*", "IDS", "POINT", 33.00005, -112.00005, 100000}, {
match("[0 [truck1 truck2]]"),
match("[0 [train1 truck1 truck2]]"),
},
{"NEARBY", "fleet", "MATCH", "t*2", "IDS", "POINT", 33.00005, -112.00005, 100000}, {
match("[0 [truck2]]"),
Expand All @@ -617,13 +622,13 @@ func keys_MATCH_test(mc *mockServer) error {
},

{"INTERSECTS", "fleet", "IDS", "BOUNDS", 33, -113, 34, -112}, {
match("[0 [luck1 luck2 truck1 truck2]]"),
match("[0 [luck1 luck2 train1 truck1 truck2]]"),
},
{"INTERSECTS", "fleet", "MATCH", "*", "IDS", "BOUNDS", 33, -113, 34, -112}, {
match("[0 [luck1 luck2 truck1 truck2]]"),
match("[0 [luck1 luck2 train1 truck1 truck2]]"),
},
{"INTERSECTS", "fleet", "MATCH", "t*", "IDS", "BOUNDS", 33, -113, 34, -112}, {
match("[0 [truck1 truck2]]"),
match("[0 [train1 truck1 truck2]]"),
},
{"INTERSECTS", "fleet", "MATCH", "t*2", "IDS", "BOUNDS", 33, -113, 34, -112}, {
match("[0 [truck2]]"),
Expand Down

0 comments on commit f24c251

Please sign in to comment.