Merge pull request #1 from LeandroSNunes/ln-sanitize

sanitize string for match_with operator

Author: LeandroSNunes

go.mod

@@ -1,3 +1,5 @@
 module github.com/enjoei/pkg
 
 go 1.13
+
+require golang.org/x/text v0.3.2

go.sum

@@ -0,0 +1,3 @@
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

querybuilder/rule.go

@@ -14,6 +14,7 @@ type Rule struct {
 	Type     string
 	Input    string
 	Operator string
+	Sanitize bool
 	Value    interface{}
 }
 
@@ -77,7 +78,13 @@ func (r *Rule) getInputValue(dataset map[string]interface{}) interface{} {
 		}
 	}
 
-	return r.parseValue(result)
+	iv := r.parseValue(result)
+	if r.Sanitize && r.Type == "string" {
+		v := iv.(string)
+		return sanitize(&v)
+	}
+
+	return iv
 }
 
 func (r *Rule) parseValue(v interface{}) interface{} {

querybuilder/rule_group.go

@@ -48,14 +48,20 @@ func (rg *RuleGroup) getChecker(rule map[string]interface{}) Checker {
 		return &RuleGroup{Condition: rule["condition"], Rules: rule["rules"]}
 	}
 
-	return &Rule{
+	r := &Rule{
 		ID:       rule["id"].(string),
 		Field:    rule["field"].(string),
 		Type:     rule["type"].(string),
 		Input:    rule["input"].(string),
 		Operator: rule["operator"].(string),
 		Value:    rule["value"],
 	}
+
+	if _, ok := rule["sanitize"]; ok {
+		r.Sanitize = rule["sanitize"].(bool)
+	}
+
+	return r
 }
 
 func (rg *RuleGroup) evaluateRules(res chan<- bool, rules []interface{}, dataset map[string]interface{}) {

querybuilder/rule_test.go

@@ -19,6 +19,7 @@ var ruleInputs = []struct {
 	{&Rule{ID: "string07", Field: "string", Type: "string", Input: "text", Operator: "greater_or_equal", Value: "my text for tests"}, true},
 	{&Rule{ID: "string08", Field: "string_empty", Type: "string", Input: "text", Operator: "is_empty", Value: "a"}, true},
 	{&Rule{ID: "string09", Field: "string", Type: "string", Input: "text", Operator: "match_with", Value: `/text\sfor/`}, true},
+	{&Rule{ID: "string10", Field: "string", Type: "string", Input: "text", Operator: "match_with", Sanitize: true, Value: `/textfor/`}, true},
 	{&Rule{ID: "double01", Field: "double", Type: "double", Input: "text", Operator: "between", Value: []interface{}{1.0, 2.0}}, true},
 	{&Rule{ID: "double02", Field: "double", Type: "double", Input: "text", Operator: "equal", Value: 1.2}, true},
 	{&Rule{ID: "double03", Field: "double", Type: "double", Input: "text", Operator: "greater", Value: 1.3}, false},

querybuilder/sanitize.go

@@ -0,0 +1,35 @@
+package querybuilder
+
+import (
+	"golang.org/x/text/runes"
+	"golang.org/x/text/transform"
+	"golang.org/x/text/unicode/norm"
+	"regexp"
+	"unicode"
+)
+
+const NoSymbolsPattern = "[^a-zA-Z0-9]+"
+
+func sanitize(s *string) string {
+	output := removeAccents(s)
+	return *removeSymbols(output)
+}
+
+func removeAccents(s *string) *string {
+	t := transform.Chain(norm.NFD, runes.Remove(runes.In(unicode.Mn)), norm.NFC)
+	output, _, err := transform.String(t, *s)
+	if err != nil {
+		return s
+	}
+	return &output
+}
+
+func removeSymbols(s *string) *string {
+	reg, err := regexp.Compile(NoSymbolsPattern)
+	if err != nil {
+		return s
+	}
+	processedString := reg.ReplaceAllString(*s, "")
+
+	return &processedString
+}

querybuilder/sanitize_test.go

@@ -0,0 +1,30 @@
+package querybuilder
+
+import "testing"
+
+func TestRemoveAccents(t *testing.T) {
+	aText := "ßàáâãäåæçèéêëìíîïðłñńòóôõōöøśùúûūüýþÿżœ"
+	rText := "ßaaaaaaæceeeeiiiiðłnnooooooøsuuuuuyþyzœ"
+
+	if text := removeAccents(&aText); *text != rText {
+		t.Errorf("removeAccents(%s) = %s -- want: %s", aText, *text, rText)
+	}
+}
+
+func TestRemoveSymbols(t *testing.T) {
+	aText := "a.b,c!?d:(e)'\"f-_g"
+	rText := "abcdefg"
+
+	if text := removeSymbols(&aText); *text != rText {
+		t.Errorf("removeSymbols(%s) = %s -- want: %s", aText, *text, rText)
+	}
+}
+
+func TestSanitize(t *testing.T) {
+	aText := "ßàáâãäåæçèéêëìíîïðłñńòóôõōöøśùúûūüýþÿżœa.b,c!?d:(e)'\"f-_g"
+	rText := "aaaaaaceeeeiiiinnoooooosuuuuuyyzabcdefg"
+
+	if text := sanitize(&aText); text != rText {
+		t.Errorf("removeSymbols(%s) = %s -- want: %s", aText, text, rText)
+	}
+}