Skip to content

Commit

Permalink
Added new Debugging API (#1896)
Browse files Browse the repository at this point in the history 
Added new Tracing API, which allows you to debug API definition, by sending sample request to it.

As input, Tracing API require sample request object, and direct API definition.
This means that tracing API can be used even without creating API. Or for example, you can test API changes without saving them.

Example request:
```
POST /tyk/trace
{
    "request": {
        "method": "GET",
        "path": "/get",
        "headers": {
            "Header": ["Value"]
        }
    },
    "spec": {
        ...valid API definition...
    }
}
```

As output, it gives you response object, and gateway logs as JSON multi-line string.

Example output:
```
{
    "message": "ok"
    "response": "<raw-http-response-dump>",
    "logs": "{...}\n{...}"
}
```

Logging itself now way more structured, and prepared for easy parsing. Based on the logs you can see:
* API loading process: which middleware was loaded or not, issues during parsing and etc.
* Middleware logs: order of running, elapsed time, individual MW logs

Example of log output:

```
{"level":"debug","msg":"Default JSVM timeout used: 5s","prefix":"jsvm","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"info","msg":"Loading API","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Initializing API","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"RateCheckMW","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"VersionCheck","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"info","msg":"Checking security policy: Token","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"AuthKey","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"KeyExpired","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"AccessRightsCheck","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"RateLimitAndQuotaCheck","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"GranularAccessMiddleware","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"VersionCheck","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"KeyExpired","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Init","mw":"AccessRightsCheck","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Rate limit endpoint is: /sampletyk/rate-limits/","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Setting Listen Path: /sample","org_id":"","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"info","msg":"API Loaded","org_id":"","prefix":"gateway","server_name":"--","time":"2018-09-05T17:49:19+03:00","user_id":"--","user_ip":"--"}
{"api_id":"test","api_name":"","level":"debug","msg":"Started","mw":"RateCheckMW","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00","ts":1536158959332155740}
{"api_id":"test","api_name":"","code":200,"level":"debug","msg":"Finished","mw":"RateCheckMW","ns":27925,"org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Started","mw":"VersionCheck","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00","ts":1536158959332214289}
{"api_id":"test","api_name":"","code":200,"level":"debug","msg":"Finished","mw":"VersionCheck","ns":23042,"org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Started","mw":"AuthKey","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00","ts":1536158959332264989}
{"api_id":"test","api_name":"","level":"debug","msg":"Querying local cache","mw":"AuthKey","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Querying keystore","mw":"AuthKey","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","level":"debug","msg":"Got key","mw":"AuthKey","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","code":200,"level":"debug","msg":"Finished","mw":"AuthKey","ns":1149266,"org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","key":"****ecf1","level":"debug","msg":"Started","mw":"KeyExpired","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00","ts":1536158959333468462}
{"api_id":"test","api_name":"","code":200,"key":"****ecf1","level":"debug","msg":"Finished","mw":"KeyExpired","ns":23915,"org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","key":"****ecf1","level":"debug","msg":"Started","mw":"AccessRightsCheck","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00","ts":1536158959333521672}
{"api_id":"test","api_name":"","code":200,"key":"****ecf1","level":"debug","msg":"Finished","mw":"AccessRightsCheck","ns":18849,"org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","key":"****ecf1","level":"debug","msg":"Started","mw":"RateLimitAndQuotaCheck","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00","ts":1536158959333632612}
{"api_id":"test","api_name":"","code":200,"key":"****ecf1","level":"debug","msg":"Finished","mw":"RateLimitAndQuotaCheck","ns":42329,"org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
{"api_id":"test","api_name":"","key":"****ecf1","level":"debug","msg":"Started","mw":"GranularAccessMiddleware","org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00","ts":1536158959333706674}
{"api_id":"test","api_name":"","code":200,"key":"****ecf1","level":"debug","msg":"Finished","mw":"GranularAccessMiddleware","ns":17514,"org_id":"","origin":"192.0.2.1","path":"/","time":"2018-09-05T17:49:19+03:00"}
```

Such detailed logs, accessed progmatically, allow some exciting cases like debugging VirtualEndpoint code, and seeing all JS parsing issues, or JS logging data, directly from Dashboard UI.

This API also allow you to test authentification as well. To do so, you just need to create key for the API, or JWT token, and provide authentification header to tracing API. So you can test full API cycle, include access rights and etc.

If your API is protected, and you do not provide token to Tracing API, you will get unauthorized error.
  • Loading branch information
@buger
buger authored Sep 9, 2018
1 parent 31f70e4 commit 9caced0
Show file tree
Hide file tree
Showing 40 changed files with 540 additions and 510 deletions.
8 changes: 8 additions & 0 deletions api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1742,3 +1742,11 @@ func ctxSetLoopLimit(r *http.Request, limit int) {
setCtxValue(r, LoopLevelLimit, limit)
}
}

func ctxTraceEnabled(r *http.Request) bool {
return r.Context().Value(Trace) != nil
}

func ctxSetTrace(r *http.Request) {
setCtxValue(r, Trace, true)
}
31 changes: 19 additions & 12 deletions api_definition.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"text/template"
"time"

"github.com/Sirupsen/logrus"
"github.com/rubyist/circuitbreaker"

"github.com/TykTechnologies/gojsonschema"
Expand Down Expand Up @@ -167,17 +168,21 @@ var ServiceNonce string

// MakeSpec will generate a flattened URLSpec from and APIDefinitions' VersionInfo data. paths are
// keyed to the Api version name, which is determined during routing to speed up lookups
func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition) *APISpec {
func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition, logger *logrus.Entry) *APISpec {
spec := &APISpec{}

if logger == nil {
logger = logrus.NewEntry(log)
}

// parse version expiration time stamps
for key, ver := range def.VersionData.Versions {
if ver.Expires == "" || ver.Expires == "-1" {
continue
}
// calculate the time
if t, err := time.Parse("2006-01-02 15:04", ver.Expires); err != nil {
log.WithError(err).WithField("Expires", ver.Expires).Error("Could not parse expiry date for API")
logger.WithError(err).WithField("Expires", ver.Expires).Error("Could not parse expiry date for API")
} else {
ver.ExpiresTs = t
def.VersionData.Versions[key] = ver
Expand All @@ -201,22 +206,24 @@ func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition) *APISpec {

// Create and init the virtual Machine
if config.Global().EnableJSVM {
spec.JSVM.Init(spec)
spec.JSVM.Init(spec, logger)
}

// Set up Event Handlers
log.Debug("INITIALISING EVENT HANDLERS")
if len(def.EventHandlers.Events) > 0 {
logger.Debug("Initializing event handlers")
}
spec.EventPaths = make(map[apidef.TykEvent][]config.TykEventHandler)
for eventName, eventHandlerConfs := range def.EventHandlers.Events {
log.Debug("FOUND EVENTS TO INIT")
logger.Debug("FOUND EVENTS TO INIT")
for _, handlerConf := range eventHandlerConfs {
log.Debug("CREATING EVENT HANDLERS")
logger.Debug("CREATING EVENT HANDLERS")
eventHandlerInstance, err := EventHandlerByName(handlerConf, spec)

if err != nil {
log.Error("Failed to init event handler: ", err)
logger.Error("Failed to init event handler: ", err)
} else {
log.Debug("Init Event Handler: ", eventName)
logger.Debug("Init Event Handler: ", eventName)
spec.EventPaths[eventName] = append(spec.EventPaths[eventName], eventHandlerInstance)
}

Expand All @@ -234,7 +241,7 @@ func (a APIDefinitionLoader) MakeSpec(def *apidef.APIDefinition) *APISpec {
pathSpecs, whiteListSpecs = a.getExtendedPathSpecs(v, spec)

} else {
log.Warning("Legacy path detected! Upgrade to extended.")
logger.Warning("Legacy path detected! Upgrade to extended.")
pathSpecs, whiteListSpecs = a.getPathSpecs(v)
}
spec.RxPaths[v.Name] = pathSpecs
Expand Down Expand Up @@ -321,7 +328,7 @@ func (a APIDefinitionLoader) FromDashboardService(endpoint, secret string) ([]*A
// Process
var specs []*APISpec
for _, def := range apiDefs {
spec := a.MakeSpec(def)
spec := a.MakeSpec(def, nil)
specs = append(specs, spec)
}

Expand Down Expand Up @@ -383,7 +390,7 @@ func (a APIDefinitionLoader) processRPCDefinitions(apiCollection string) []*APIS
def.Proxy.ListenPath = newListenPath
}

spec := a.MakeSpec(def)
spec := a.MakeSpec(def, nil)
specs = append(specs, spec)
}

Expand Down Expand Up @@ -413,7 +420,7 @@ func (a APIDefinitionLoader) FromDir(dir string) []*APISpec {
}
def := a.ParseDefinition(f)
f.Close()
spec := a.MakeSpec(def)
spec := a.MakeSpec(def, nil)
specs = append(specs, spec)
}
return specs
Expand Down
2 changes: 1 addition & 1 deletion api_definition_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (
func createDefinitionFromString(defStr string) *APISpec {
loader := APIDefinitionLoader{}
def := loader.ParseDefinition(strings.NewReader(defStr))
spec := loader.MakeSpec(def)
spec := loader.MakeSpec(def, nil)
return spec
}

Expand Down
111 changes: 43 additions & 68 deletions api_loader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,30 +46,21 @@ func prepareStorage() (storage.RedisCluster, storage.RedisCluster, storage.Redis
return redisStore, redisOrgStore, healthStore, &rpcAuthStore, &rpcOrgStore
}

func skipSpecBecauseInvalid(spec *APISpec) bool {
func skipSpecBecauseInvalid(spec *APISpec, logger *logrus.Entry) bool {

if spec.Proxy.ListenPath == "" {
mainLog.WithFields(logrus.Fields{
"org_id": spec.OrgID,
"api_id": spec.APIID,
}).Error("Listen path is empty, skipping API ID: ", spec.APIID)
logger.Error("Listen path is empty")
return true
}

if strings.Contains(spec.Proxy.ListenPath, " ") {
mainLog.WithFields(logrus.Fields{
"org_id": spec.OrgID,
"api_id": spec.APIID,
}).Error("Listen path contains spaces, is invalid, skipping API ID: ", spec.APIID)
logger.Error("Listen path contains spaces, is invalid")
return true
}

_, err := url.Parse(spec.Proxy.TargetURL)
if err != nil {
mainLog.WithFields(logrus.Fields{
"org_id": spec.OrgID,
"api_id": spec.APIID,
}).Error("couldn't parse target URL: ", err)
logger.Error("couldn't parse target URL: ", err)
return true
}

Expand Down Expand Up @@ -102,19 +93,22 @@ func countApisByListenHash(specs []*APISpec) map[string]int {

func processSpec(spec *APISpec, apisByListen map[string]int,
redisStore, redisOrgStore, healthStore, rpcAuthStore, rpcOrgStore storage.Handler,
subrouter *mux.Router) *ChainObject {
subrouter *mux.Router, logger *logrus.Entry) *ChainObject {

var chainDef ChainObject
chainDef.Subrouter = subrouter

var coprocessLog = log.WithFields(logrus.Fields{
"prefix": "coprocess",
logger = logger.WithFields(logrus.Fields{
"org_id": spec.OrgID,
"api_id": spec.APIID,
"api_name": spec.Name,
})

mainLog.WithFields(logrus.Fields{
"api_name": spec.Name,
}).Info("Loading API")
var coprocessLog = logger.WithFields(logrus.Fields{
"prefix": "coprocess",
})

logger.Info("Loading API")

if len(spec.TagHeaders) > 0 {
// Ensure all headers marked for tagging are lowercase
Expand All @@ -126,10 +120,8 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
spec.TagHeaders = lowerCaseHeaders
}

if skipSpecBecauseInvalid(spec) {
mainLog.WithFields(logrus.Fields{
"api_name": spec.Name,
}).Warning("Skipped!")
if skipSpecBecauseInvalid(spec, logger) {
logger.Warning("Spec not valid, skipped!")
chainDef.Skip = true
return &chainDef
}
Expand All @@ -156,10 +148,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
}
}
if pathModified {
mainLog.WithFields(logrus.Fields{
"org_id": spec.OrgID,
"api_id": spec.APIID,
}).Error("Listen path collision, changed to ", spec.Proxy.ListenPath)
logger.Error("Listen path collision, changed to ", spec.Proxy.ListenPath)
}

// Set up LB targets:
Expand Down Expand Up @@ -205,9 +194,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
var prefix string
if spec.CustomMiddlewareBundle != "" {
if err := loadBundle(spec); err != nil {
mainLog.WithFields(logrus.Fields{
"api_name": spec.Name,
}).Error("Couldn't load bundle")
logger.Error("Couldn't load bundle")
}
tykBundlePath := filepath.Join(config.Global().MiddlewarePath, "bundles")
bundleNameHash := md5.New()
Expand All @@ -216,9 +203,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
prefix = filepath.Join(tykBundlePath, bundlePath)
}

mainLog.WithFields(logrus.Fields{
"api_name": spec.Name,
}).Debug("Loading Middleware")
logger.Debug("Initializing API")
var mwPaths []string

mwPaths, mwAuthCheckFunc, mwPreFuncs, mwPostFuncs, mwPostAuthCheckFuncs, mwDriver = loadCustomMiddleware(spec)
Expand All @@ -232,12 +217,12 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
}

if spec.UseOauth2 {
log.Debug("Loading OAuth Manager")
logger.Debug("Loading OAuth Manager")
oauthManager := addOAuthHandlers(spec, subrouter)
log.Debug("-- Added OAuth Handlers")
logger.Debug("-- Added OAuth Handlers")

spec.OAuthManager = oauthManager
log.Debug("Done loading OAuth Manager")
logger.Debug("Done loading OAuth Manager")
}

enableVersionOverrides := false
Expand All @@ -253,9 +238,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,

var proxy ReturningHttpHandler
if enableVersionOverrides {
mainLog.WithFields(logrus.Fields{
"api_name": spec.Name,
}).Info("Multi target enabled")
logger.Info("Multi target enabled")
proxy = NewMultiTargetProxy(spec)
} else {
proxy = TykNewSingleHostReverseProxy(spec.target, spec)
Expand All @@ -264,7 +247,8 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
// Create the response processors
createResponseMiddlewareChain(spec)

baseMid := BaseMiddleware{spec, proxy}
baseMid := BaseMiddleware{Spec: spec, Proxy: proxy, logger: logger}

for _, v := range baseMid.Spec.VersionData.Versions {
if len(v.ExtendedPaths.CircuitBreaker) > 0 {
baseMid.Spec.CircuitBreakerEnabled = true
Expand All @@ -282,9 +266,7 @@ func processSpec(spec *APISpec, apisByListen map[string]int,

if spec.UseKeylessAccess {
chainDef.Open = true
mainLog.WithFields(logrus.Fields{
"api_name": spec.Name,
}).Info("Checking security policy: Open")
logger.Info("Checking security policy: Open")

// Add pre-process MW
chainArray := []alice.Constructor{}
Expand Down Expand Up @@ -362,23 +344,23 @@ func processSpec(spec *APISpec, apisByListen map[string]int,

// Select the keying method to use for setting session states
if mwAppendEnabled(&authArray, &Oauth2KeyExists{baseMid}) {
mainLog.WithField("api_name", spec.Name).Info("Checking security policy: OAuth")
logger.Info("Checking security policy: OAuth")
}

if mwAppendEnabled(&authArray, &BasicAuthKeyIsValid{baseMid, cache.New(60*time.Second, 60*time.Minute)}) {
mainLog.WithField("api_name", spec.Name).Info("Checking security policy: Basic")
logger.Info("Checking security policy: Basic")
}

if mwAppendEnabled(&authArray, &HMACMiddleware{BaseMiddleware: baseMid}) {
mainLog.WithField("api_name", spec.Name).Info("Checking security policy: HMAC")
logger.Info("Checking security policy: HMAC")
}

if mwAppendEnabled(&authArray, &JWTMiddleware{baseMid}) {
mainLog.WithField("api_name", spec.Name).Info("Checking security policy: JWT")
logger.Info("Checking security policy: JWT")
}

if mwAppendEnabled(&authArray, &OpenIDMW{BaseMiddleware: baseMid}) {
mainLog.WithField("api_name", spec.Name).Info("Checking security policy: OpenID")
logger.Info("Checking security policy: OpenID")
}

coprocessAuth := EnableCoProcess && mwDriver != apidef.OttoDriver && spec.EnableCoProcessAuth
Expand All @@ -394,13 +376,13 @@ func processSpec(spec *APISpec, apisByListen map[string]int,

if ottoAuth {

mainLog.Info("----> Checking security policy: JS Plugin")
logger.Info("----> Checking security policy: JS Plugin")

authArray = append(authArray, createDynamicMiddleware(mwAuthCheckFunc.Name, true, false, baseMid))
}

if spec.UseStandardAuth || len(authArray) == 0 {
mainLog.WithField("api_name", spec.Name).Info("Checking security policy: Token")
logger.Info("Checking security policy: Token")
authArray = append(authArray, createMiddleware(&AuthKey{baseMid}))
}

Expand Down Expand Up @@ -437,13 +419,9 @@ func processSpec(spec *APISpec, apisByListen map[string]int,
}
}

mainLog.WithField("api_name", spec.Name).Debug("Custom middleware completed processing")

// Use createMiddleware(&ModifiedMiddleware{baseMid}) to run custom middleware
chain = alice.New(chainArray...).Then(&DummyProxyHandler{SH: SuccessHandler{baseMid}})

log.Debug("Chain completed")

var simpleArray []alice.Constructor
mwAppendEnabled(&simpleArray, &IPWhiteListMiddleware{baseMid})
mwAppendEnabled(&simpleArray, &IPBlackListMiddleware{BaseMiddleware: baseMid})
Expand All @@ -455,28 +433,24 @@ func processSpec(spec *APISpec, apisByListen map[string]int,

rateLimitPath := spec.Proxy.ListenPath + "tyk/rate-limits/"

mainLog.WithField("api_name", spec.Name).Debug("Rate limit endpoint is: ", rateLimitPath)
logger.Debug("Rate limit endpoint is: ", rateLimitPath)

chainDef.RateLimitPath = rateLimitPath
chainDef.RateLimitChain = alice.New(simpleArray...).
Then(http.HandlerFunc(userRatesCheck))
}

mainLog.WithField("api_name", spec.Name).Debug("Setting Listen Path: ", spec.Proxy.ListenPath)
logger.Debug("Setting Listen Path: ", spec.Proxy.ListenPath)

chainDef.ThisHandler = chain
chainDef.ListenOn = spec.Proxy.ListenPath + "{rest:.*}"

if config.Global().UseRedisLog {
log.WithFields(logrus.Fields{
"prefix": "gateway",
"user_ip": "--",
"server_name": "--",
"user_id": "--",
"org_id": spec.OrgID,
"api_id": spec.APIID,
}).Info("Loaded: ", spec.Name)
}
logger.WithFields(logrus.Fields{
"prefix": "gateway",
"user_ip": "--",
"server_name": "--",
"user_id": "--",
}).Info("API Loaded")

return &chainDef
}
Expand Down Expand Up @@ -508,7 +482,7 @@ type DummyProxyHandler struct {
func (d *DummyProxyHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if found, err := isLoop(r); found {
if err != nil {
handler := ErrorHandler{d.SH.Base()}
handler := ErrorHandler{*d.SH.Base()}
handler.HandleError(w, r, err.Error(), http.StatusInternalServerError)
return
}
Expand Down Expand Up @@ -618,7 +592,8 @@ func loadApps(specs []*APISpec, muxer *mux.Router) {
subrouter = muxer
}

chainObj := processSpec(spec, apisByListen, redisStore, redisOrgStore, healthStore, rpcAuthStore, rpcOrgStore, subrouter)
chainObj := processSpec(spec, apisByListen, redisStore, redisOrgStore, healthStore, rpcAuthStore, rpcOrgStore, subrouter, logrus.NewEntry(log))

chainObj.Index = i
chainChannel <- chainObj
apisMu.Lock()
Expand Down
Loading

0 comments on commit 9caced0

Please sign in to comment.