Skip to content

Commit

Permalink
Refactor and speedup url-rewrite (#1511)
Browse files Browse the repository at this point in the history 
- Added new `replaceTykVariables` function, which replaces all
`$tyk_context` and `$tyk_meta` variables in string`
- URLRewrite regexp now cached. Previously it was compiling on each
call.
  • Loading branch information
@buger
buger authored Mar 6, 2018
1 parent 40bc2c0 commit 4a7e0ac
Show file tree
Hide file tree
Showing 3 changed files with 83 additions and 115 deletions.
1 change: 1 addition & 0 deletions apidef/api_definitions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ type URLRewriteMeta struct {
MatchPattern string `bson:"match_pattern" json:"match_pattern"`
RewriteTo string `bson:"rewrite_to" json:"rewrite_to"`
Triggers []RoutingTrigger `bson:"triggers" json:"triggers"`
MatchRegexp *regexp.Regexp
}

type VirtualMeta struct {
Expand Down
57 changes: 5 additions & 52 deletions mw_modify_headers.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
package main

import (
"fmt"
"net/http"
"strings"

"github.com/TykTechnologies/tyk/apidef"
)
Expand All @@ -13,11 +11,6 @@ type TransformHeaders struct {
BaseMiddleware
}

const (
metaLabel = "$tyk_meta."
contextLabel = "$tyk_context."
)

func (t *TransformHeaders) Name() string {
return "TransformHeaders"
}
Expand All @@ -33,48 +26,6 @@ func (t *TransformHeaders) EnabledForSpec() bool {
return false
}

// iterateAddHeaders is a helper functino that will iterate of a map and inject the key and value as a header in the request.
// if the key and value contain a tyk session variable reference, then it will try to inject the value
func (t *TransformHeaders) iterateAddHeaders(kv map[string]string, r *http.Request) {
// Get session data
session := ctxGetSession(r)

contextData := ctxGetData(r)

// Iterate and manage key array injection
for nKey, nVal := range kv {
if strings.Contains(nVal, metaLabel) {
// Using meta_data key
if session != nil {
metaKey := strings.Replace(nVal, metaLabel, "", 1)
tempVal, ok := session.MetaData[metaKey]
if ok {
// TODO: do a better job than fmt's %v
nVal = fmt.Sprintf("%v", tempVal)
r.Header.Set(nKey, nVal)
} else {
log.Warning("Session Meta Data not found for key in map: ", metaKey)
}
}

} else if strings.Contains(nVal, contextLabel) {
// Using context key
if contextData != nil {
metaKey := strings.Replace(nVal, contextLabel, "", 1)
val, ok := contextData[metaKey]
if ok {
r.Header.Set(nKey, valToStr(val))
} else {
log.Warning("Context Data not found for key in map: ", metaKey)
}
}

} else {
r.Header.Set(nKey, nVal)
}
}
}

// ProcessRequest will run any checks on the request on the way through the system, return an error to have the chain fail
func (t *TransformHeaders) ProcessRequest(w http.ResponseWriter, r *http.Request, _ interface{}) (error, int) {
vInfo, versionPaths, _, _ := t.Spec.Version(r)
Expand All @@ -86,8 +37,8 @@ func (t *TransformHeaders) ProcessRequest(w http.ResponseWriter, r *http.Request
}

// Add
if len(vInfo.GlobalHeaders) > 0 {
t.iterateAddHeaders(vInfo.GlobalHeaders, r)
for nKey, nVal := range vInfo.GlobalHeaders {
r.Header.Set(nKey, replaceTykVariables(r, nVal, false))
}

found, meta := t.Spec.CheckSpecMatchesStatus(r, versionPaths, HeaderInjected)
Expand All @@ -96,7 +47,9 @@ func (t *TransformHeaders) ProcessRequest(w http.ResponseWriter, r *http.Request
for _, dKey := range hmeta.DeleteHeaders {
r.Header.Del(dKey)
}
t.iterateAddHeaders(hmeta.AddHeaders, r)
for nKey, nVal := range hmeta.AddHeaders {
r.Header.Set(nKey, replaceTykVariables(r, nVal, false))
}
}

return nil, 200
Expand Down
140 changes: 77 additions & 63 deletions mw_url_rewrite.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,27 @@ import (
"github.com/TykTechnologies/tyk/user"
)

const (
metaLabel = "$tyk_meta."
contextLabel = "$tyk_context."
)

var dollarMatch = regexp.MustCompile(`\$\d+`)
var contextMatch = regexp.MustCompile(`\$tyk_context.([A-Za-z0-9_\-\.]+)`)
var metaMatch = regexp.MustCompile(`\$tyk_meta.([A-Za-z0-9_\-\.]+)`)

func urlRewrite(meta *apidef.URLRewriteMeta, r *http.Request) (string, error) {
// Find all the matching groups:
mp, err := regexp.Compile(meta.MatchPattern)
if err != nil {
log.Debug("Compilation error: ", err)
return "", err
}
path := r.URL.String()
log.Debug("Inbound path: ", path)
newpath := path

result_slice := mp.FindAllStringSubmatch(path, -1)
if meta.MatchRegexp == nil {
var err error
meta.MatchRegexp, err = regexp.Compile(meta.MatchPattern)
if err != nil {
return path, fmt.Errorf("URLRewrite regexp error %s", meta.MatchPattern)
}
}

// Check triggers
rewriteToPath := meta.RewriteTo
Expand Down Expand Up @@ -130,78 +139,84 @@ func urlRewrite(meta *apidef.URLRewriteMeta, r *http.Request) (string, error) {
}
}

matchGroups := meta.MatchRegexp.FindAllStringSubmatch(path, -1)

// Make sure it matches the string
log.Debug("Rewriter checking matches, len is: ", len(result_slice))
if len(result_slice) > 0 {
log.Debug("Rewriter checking matches, len is: ", len(matchGroups))
if len(matchGroups) > 0 {
newpath = rewriteToPath
// get the indices for the replacements:
dollarMatch := regexp.MustCompile(`\$\d+`) // Prepare our regex
replace_slice := dollarMatch.FindAllStringSubmatch(rewriteToPath, -1)
replaceGroups := dollarMatch.FindAllStringSubmatch(rewriteToPath, -1)

log.Debug(result_slice)
log.Debug(replace_slice)
log.Debug(matchGroups)
log.Debug(replaceGroups)

mapped_replace := make(map[string]string)
for mI, replacementVal := range result_slice[0] {
groupReplace := make(map[string]string)
for mI, replacementVal := range matchGroups[0] {
indexVal := "$" + strconv.Itoa(mI)
mapped_replace[indexVal] = replacementVal
groupReplace[indexVal] = replacementVal
}

for _, v := range replace_slice {
log.Debug("Replacing: ", v[0])
newpath = strings.Replace(newpath, v[0], mapped_replace[v[0]], -1)
for _, v := range replaceGroups {
newpath = strings.Replace(newpath, v[0], groupReplace[v[0]], -1)
}

log.Debug("URL Re-written from: ", path)
log.Debug("URL Re-written to: ", newpath)

// put url_rewrite path to context to be used in ResponseTransformMiddleware
ctxSetUrlRewritePath(r, meta.Path)

// matched?? Set the modified path
// return newpath, nil
}

contextData := ctxGetData(r)
newpath = replaceTykVariables(r, newpath, true)

return newpath, nil
}

dollarMatch := regexp.MustCompile(`\$tyk_context.([A-Za-z0-9_\-\.]+)`)
replace_slice := dollarMatch.FindAllStringSubmatch(rewriteToPath, -1)
for _, v := range replace_slice {
contextKey := strings.Replace(v[0], "$tyk_context.", "", 1)
func replaceTykVariables(r *http.Request, in string, escape bool) string {
if strings.Contains(in, contextLabel) {
contextData := ctxGetData(r)

replaceGroups := contextMatch.FindAllStringSubmatch(in, -1)
for _, v := range replaceGroups {
contextKey := strings.Replace(v[0], "$tyk_context.", "", 1)

if val, ok := contextData[contextKey]; ok {
valStr := valToStr(val)
// If contains url with domain
if !strings.HasPrefix(valStr, "http") {
valStr = url.QueryEscape(valStr)
if val, ok := contextData[contextKey]; ok {
valStr := valToStr(val)
// If contains url with domain
if escape && !strings.HasPrefix(valStr, "http") {
valStr = url.QueryEscape(valStr)
}
in = strings.Replace(in, v[0], valStr, -1)
}
newpath = strings.Replace(newpath, v[0], valStr, -1)
}
}

// Meta data from the token
if session := ctxGetSession(r); session != nil {
if strings.Contains(in, metaLabel) {
// Meta data from the token
session := ctxGetSession(r)
if session == nil {
return in
}

metaDollarMatch := regexp.MustCompile(`\$tyk_meta.(\w+)`)
metaReplace_slice := metaDollarMatch.FindAllStringSubmatch(rewriteToPath, -1)
for _, v := range metaReplace_slice {
replaceGroups := metaMatch.FindAllStringSubmatch(in, -1)
for _, v := range replaceGroups {
contextKey := strings.Replace(v[0], "$tyk_meta.", "", 1)
log.Debug("Replacing: ", v[0])

val, ok := session.MetaData[contextKey]
if ok {
valStr := valToStr(val)
// If contains url with domain
if !strings.HasPrefix(valStr, "http") {
if escape && !strings.HasPrefix(valStr, "http") {
valStr = url.QueryEscape(valStr)
}
newpath = strings.Replace(newpath, v[0], valStr, -1)
in = strings.Replace(in, v[0], valStr, -1)
}

}
}

return newpath, nil
return in
}

func valToStr(v interface{}) string {
Expand Down Expand Up @@ -241,37 +256,35 @@ func (m *URLRewriteMiddleware) InitTriggerRx() {
// Generate regexp for each special match parameter
for verKey := range m.Spec.VersionData.Versions {
for pathKey := range m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite {
for trKey := range m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].Triggers {
for key, h := range m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.HeaderMatches {
rewrite := m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey]

for trKey := range rewrite.Triggers {
tr := rewrite.Triggers[trKey]

for key, h := range tr.Options.HeaderMatches {
h.Init()
m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.HeaderMatches[key] = h
tr.Options.HeaderMatches[key] = h
}
for key, q := range m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.QueryValMatches {
for key, q := range tr.Options.QueryValMatches {
q.Init()
m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.QueryValMatches[key] = q
tr.Options.QueryValMatches[key] = q
}
for key, h := range m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.SessionMetaMatches {
for key, h := range tr.Options.SessionMetaMatches {
h.Init()
m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.SessionMetaMatches[key] = h
tr.Options.SessionMetaMatches[key] = h
}
for key, h := range m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.PathPartMatches {
for key, h := range tr.Options.PathPartMatches {
h.Init()
m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.PathPartMatches[key] = h
tr.Options.PathPartMatches[key] = h
}
if m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.PayloadMatches.MatchPattern != "" {
m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey].
Triggers[trKey].Options.PayloadMatches.Init()
if tr.Options.PayloadMatches.MatchPattern != "" {
tr.Options.PayloadMatches.Init()
}

rewrite.Triggers[trKey] = tr
}

m.Spec.VersionData.Versions[verKey].ExtendedPaths.URLRewrite[pathKey] = rewrite
}
}
}
Expand Down Expand Up @@ -310,6 +323,7 @@ func (m *URLRewriteMiddleware) ProcessRequest(w http.ResponseWriter, r *http.Req
oldPath := r.URL.String()
p, err := urlRewrite(umeta, r)
if err != nil {
log.Error(err)
return err, 500
}

Expand Down

0 comments on commit 4a7e0ac

Please sign in to comment.