diff --git a/.goreleaser.yml b/.goreleaser.yml index cd48d333..20ee3db3 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -185,13 +185,7 @@ brews: # in case there is an indicator for prerelease in the tag e.g. v1.0.0-rc1 # # Templates: allowed - skip_upload: auto - - # Custom block for brew. - # Can be used to specify alternate downloads for devel or head releases. - custom_block: | - head "https://github.com/some/package.git" - ... + skip_upload: false # auto # Packages your package depends on. dependencies: [] @@ -225,7 +219,6 @@ brews: # # Templates: allowed url: 'git@github.com:EinStack/homebrew-tap.git' - private_key: '{{ .Env.BREW_TAP_PRIVATE_KEY }}' announce: diff --git a/CHANGELOG.md b/CHANGELOG.md index eb05365e..550a2919 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,20 +5,28 @@ The changelog consists of three categories: - **Improvements** - bugfixes, performance and other types of improvements to existing functionality - **Miscellaneous** - all other updates like build, release, CLI, etc. +## 0.0.1-rc.2 (Jan 22nd, 2024) + +### Improvements + +- ⚙️ [config] Added validation for config file content #40 (@roma-glushko) +- ⚙️ [config] Allowed to pass HTTP server configs from config file #41 (@roma-glushko) +- 👷 [build] Allowed building Homebrew taps for release candidates #99 (@roma-glushko) + ## 0.0.1-rc.1 (Jan 21st, 2024) ### Features -- ✨ [providers] Support for OpenAI Chat API #3 (@mkrueger12 ) -- ✨ [API] #54 Unified Chat API (@mkrueger12 ) -- ✨ [providers] Support for Cohere Chat API #5 (@mkrueger12 ) -- ✨ [providers] Support for Azure OpenAI Chat API #4 (@mkrueger12 ) -- ✨ [providers] Support for OctoML Chat API #58 (@mkrueger12 ) +- ✨ [providers] Support for OpenAI Chat API #3 (@mkrueger12) +- ✨ [API] Unified Chat API #54 (@mkrueger12) +- ✨ [providers] Support for Cohere Chat API #5 (@mkrueger12) +- ✨ [providers] Support for Azure OpenAI Chat API #4 (@mkrueger12) +- ✨ [providers] Support for OctoML Chat API #58 (@mkrueger12) - ✨ [routing] The Routing Mechanism, Adaptive Health Tracking, and Fallbacks #42 #43 #51 (@roma-glushko) - ✨ [routing] Support for round robin routing strategy #44 (@roma-glushko) - ✨ [routing] Support for the least latency routing strategy #46 (@roma-glushko) - ✨ [routing] Support for weighted round robin routing strategy #45 (@roma-glushko) -- ✨ [providers] Support for Anthropic Chat API #60 (@mkrueger12 ) -- ✨ [docs] OpenAPI specifications #22 (@roma-glushko ) +- ✨ [providers] Support for Anthropic Chat API #60 (@mkrueger12) +- ✨ [docs] OpenAPI specifications #22 (@roma-glushko) ### Miscellaneous diff --git a/README.md b/README.md index 9df7c083..f30bad23 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ Once deployed, Glide comes with OpenAPI documentation that is accessible via htt ## Community -- Join [Discord](https://discord.gg/z4DmAbJP) for real-time discussion +- Join [Discord](https://discord.gg/pt53Ej7rrc) for real-time discussion Open [an issue](https://github.com/modelgateway/glide/issues) or start [a discussion](https://github.com/modelgateway/glide/discussions) if there is a feature or an enhancement you'd like to see in Glide. diff --git a/config.dev.yaml b/config.dev.yaml index 30f83e87..ec07d594 100644 --- a/config.dev.yaml +++ b/config.dev.yaml @@ -3,10 +3,6 @@ telemetry: level: debug # debug, info, warn, error, fatal encoding: console -#api: -# http: -# ... - routers: language: - id: myrouter @@ -18,4 +14,4 @@ routers: azureopenai: api_key: "" model: "" - base_url: "" \ No newline at end of file + base_url: "" diff --git a/docs/docs.go b/docs/docs.go index 8a51f1ac..f5a9cc48 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -506,6 +506,7 @@ const docTemplate = `{ "providers.LangModelConfig": { "type": "object", "required": [ + "enabled", "id" ], "properties": { @@ -539,7 +540,12 @@ const docTemplate = `{ "$ref": "#/definitions/octoml.Config" }, "openai": { - "$ref": "#/definitions/openai.Config" + "description": "Add other providers like", + "allOf": [ + { + "$ref": "#/definitions/openai.Config" + } + ] }, "weight": { "type": "integer" @@ -566,8 +572,11 @@ const docTemplate = `{ "routers.LangRouterConfig": { "type": "object", "required": [ + "enabled", "models", - "routers" + "retry", + "routers", + "strategy" ], "properties": { "enabled": { @@ -577,6 +586,7 @@ const docTemplate = `{ "models": { "description": "the list of models that could handle requests", "type": "array", + "minItems": 1, "items": { "$ref": "#/definitions/providers.LangModelConfig" } diff --git a/docs/swagger.json b/docs/swagger.json index 4722323c..c6bf90fd 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -503,6 +503,7 @@ "providers.LangModelConfig": { "type": "object", "required": [ + "enabled", "id" ], "properties": { @@ -536,7 +537,12 @@ "$ref": "#/definitions/octoml.Config" }, "openai": { - "$ref": "#/definitions/openai.Config" + "description": "Add other providers like", + "allOf": [ + { + "$ref": "#/definitions/openai.Config" + } + ] }, "weight": { "type": "integer" @@ -563,8 +569,11 @@ "routers.LangRouterConfig": { "type": "object", "required": [ + "enabled", "models", - "routers" + "retry", + "routers", + "strategy" ], "properties": { "enabled": { @@ -574,6 +583,7 @@ "models": { "description": "the list of models that could handle requests", "type": "array", + "minItems": 1, "items": { "$ref": "#/definitions/providers.LangModelConfig" } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 74616bb9..0ff8a476 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -279,10 +279,13 @@ definitions: octoml: $ref: '#/definitions/octoml.Config' openai: - $ref: '#/definitions/openai.Config' + allOf: + - $ref: '#/definitions/openai.Config' + description: Add other providers like weight: type: integer required: + - enabled - id type: object retry.ExpRetryConfig: @@ -305,6 +308,7 @@ definitions: description: the list of models that could handle requests items: $ref: '#/definitions/providers.LangModelConfig' + minItems: 1 type: array retry: allOf: @@ -317,8 +321,11 @@ definitions: description: strategy on picking the next model to serve the request type: string required: + - enabled - models + - retry - routers + - strategy type: object schemas.ChatMessage: properties: diff --git a/go.mod b/go.mod index 5db4005e..d59105cb 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21.5 require ( github.com/cloudwego/hertz v0.7.3 + github.com/go-playground/validator/v10 v10.17.0 github.com/hertz-contrib/logger/zap v1.1.0 github.com/hertz-contrib/swagger v0.1.0 github.com/spf13/cobra v1.8.0 @@ -28,14 +29,18 @@ require ( github.com/cloudwego/netpoll v0.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/spec v0.20.13 // indirect github.com/go-openapi/swag v0.22.7 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/nyaruka/phonenumbers v1.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -45,6 +50,7 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect golang.org/x/arch v0.6.0 // indirect + golang.org/x/crypto v0.16.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index b8dd2d9d..7c3d524b 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= @@ -43,6 +45,14 @@ github.com/go-openapi/spec v0.20.13 h1:XJDIN+dLH6vqXgafnl5SUIMnzaChQ6QTo0/UPMbkI github.com/go-openapi/spec v0.20.13/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74= +github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -71,6 +81,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/nyaruka/phonenumbers v1.0.55/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= @@ -99,6 +111,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= @@ -129,6 +142,8 @@ golang.org/x/arch v0.6.0 h1:S0JTfE48HbRj80+4tbvZDYsJ3tGv6BUU3XxyZ7CirAc= golang.org/x/arch v0.6.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= diff --git a/main.go b/main.go index b37e583f..eaa16bdd 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,6 @@ func main() { cli := cmd.NewCLI() if err := cli.Execute(); err != nil { - log.Fatalf("glide run finished with error: %v", err) + log.Fatalf("💥Glide has finished with error: %v", err) } } diff --git a/pkg/api/config.go b/pkg/api/config.go index 6f230344..c1edbb20 100644 --- a/pkg/api/config.go +++ b/pkg/api/config.go @@ -4,7 +4,7 @@ import "glide/pkg/api/http" // Config defines configuration for all API types we support (e.g. HTTP, gRPC) type Config struct { - HTTP *http.ServerConfig `yaml:"http"` + HTTP *http.ServerConfig `yaml:"http" validate:"required"` } func DefaultConfig() *Config { diff --git a/pkg/api/http/config.go b/pkg/api/http/config.go index 1e869db1..f33cc15b 100644 --- a/pkg/api/http/config.go +++ b/pkg/api/http/config.go @@ -1,28 +1,66 @@ package http import ( + "fmt" "time" + "github.com/cloudwego/hertz/pkg/common/config" + "github.com/cloudwego/hertz/pkg/app/server" "github.com/cloudwego/hertz/pkg/network/netpoll" ) type ServerConfig struct { - HostPort string + Host string `yaml:"host"` + Port int `yaml:"port"` + ReadTimeout *time.Duration `yaml:"read_timeout"` + WriteTimeout *time.Duration `yaml:"write_timeout"` + IdleTimeout *time.Duration `yaml:"idle_timeout"` + MaxRequestBodySize *int `yaml:"max_request_body_size"` } func DefaultServerConfig() *ServerConfig { + maxReqBodySize := 4 * 1024 * 1024 + readTimeout := 3 * time.Second + writeTimeout := 3 * time.Second + idleTimeout := 1 * time.Second + return &ServerConfig{ - HostPort: "127.0.0.1:9099", + Host: "127.0.0.1", + Port: 9099, + IdleTimeout: &idleTimeout, + ReadTimeout: &readTimeout, + WriteTimeout: &writeTimeout, + MaxRequestBodySize: &maxReqBodySize, } } +func (cfg *ServerConfig) Address() string { + return fmt.Sprintf("%s:%v", cfg.Host, cfg.Port) +} + func (cfg *ServerConfig) ToServer() *server.Hertz { - // TODO: do real server build based on provided config - return server.Default( - server.WithIdleTimeout(1*time.Second), - server.WithHostPorts(cfg.HostPort), - server.WithMaxRequestBodySize(20<<20), + // More configs are listed on https://www.cloudwego.io/docs/hertz/tutorials/basic-feature/engine/ + serverOptions := []config.Option{ + server.WithHostPorts(cfg.Address()), server.WithTransport(netpoll.NewTransporter), - ) + } + + if cfg.IdleTimeout != nil { + serverOptions = append(serverOptions, server.WithIdleTimeout(*cfg.IdleTimeout)) + } + + if cfg.ReadTimeout != nil { + serverOptions = append(serverOptions, server.WithReadTimeout(*cfg.ReadTimeout)) + } + + if cfg.WriteTimeout != nil { + serverOptions = append(serverOptions, server.WithWriteTimeout(*cfg.WriteTimeout)) + } + + if cfg.MaxRequestBodySize != nil { + serverOptions = append(serverOptions, server.WithMaxRequestBodySize(*cfg.MaxRequestBodySize)) + } + + return server.Default(serverOptions...) } diff --git a/pkg/api/http/server.go b/pkg/api/http/server.go index fc274b91..f8a78d17 100644 --- a/pkg/api/http/server.go +++ b/pkg/api/http/server.go @@ -42,7 +42,7 @@ func (srv *Server) Run() error { defaultGroup.GET("/health/", HealthHandler) - schemaDocURL := swagger.URL(fmt.Sprintf("http://%v/v1/swagger/doc.json", srv.config.HostPort)) + schemaDocURL := swagger.URL(fmt.Sprintf("http://%v/v1/swagger/doc.json", srv.config.Address())) defaultGroup.GET("/swagger/*any", swagger.WrapHandler(swaggerFiles.Handler, schemaDocURL)) return srv.server.Run() diff --git a/pkg/cmd/cli.go b/pkg/cmd/cli.go index 885ec2e3..7a2cbcc0 100644 --- a/pkg/cmd/cli.go +++ b/pkg/cmd/cli.go @@ -30,7 +30,8 @@ func NewCLI() *cobra.Command { return gateway.Run(cmd.Context()) }, - // SilenceUsage: true, + SilenceUsage: true, + SilenceErrors: true, } cli.PersistentFlags().StringVarP(&cfgFile, "config", "c", "", "config file") diff --git a/pkg/config/config.go b/pkg/config/config.go index 80208bee..e792688e 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -8,8 +8,8 @@ import ( // Config is a general top-level Glide configuration type Config struct { - Telemetry *telemetry.Config `yaml:"telemetry"` - API *api.Config `yaml:"api"` + Telemetry *telemetry.Config `yaml:"telemetry" validate:"required"` + API *api.Config `yaml:"api" validate:"required"` Routers routers.Config `yaml:"routers" validate:"required"` } diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 03d4f7f9..da0856d9 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -4,21 +4,39 @@ import ( "fmt" "os" "path/filepath" + "reflect" + "strings" + + "github.com/go-playground/validator/v10" "gopkg.in/yaml.v3" ) // Provider reads, collects, validates and process config files type Provider struct { - expander *Expander - Config *Config + expander *Expander + Config *Config + validator *validator.Validate } // NewProvider creates a instance of Config Provider func NewProvider() *Provider { + configValidator := validator.New(validator.WithRequiredStructEnabled()) + + configValidator.RegisterTagNameFunc(func(fld reflect.StructField) string { + name := strings.SplitN(fld.Tag.Get("yaml"), ",", 2)[0] + + if name == "-" { + return "" + } + + return name + }) + return &Provider{ - expander: &Expander{}, - Config: nil, + expander: &Expander{}, + Config: nil, + validator: configValidator, } } @@ -38,16 +56,78 @@ func (p *Provider) Load(configPath string) (*Provider, error) { return p, fmt.Errorf("unable to parse config file %v: %w", configPath, err) } - // TODO: validate config values + err = p.validator.Struct(cfg) + + if err != nil { + return p, p.formatValidationError(configPath, err) + } p.Config = cfg return p, nil } +func (p *Provider) formatValidationError(configPath string, err error) error { + // this check is only needed when your code could produce + // an invalid value for validation such as interface with nil + // value most including myself do not usually have code like this. + if _, ok := err.(*validator.InvalidValidationError); ok { + return fmt.Errorf("invalid config file %v: %v", configPath, err) + } + + errors := make([]string, 0, len(err.(validator.ValidationErrors))) + + for _, fieldErr := range err.(validator.ValidationErrors) { + errors = append( + errors, + fmt.Sprintf( + "- ❌ %v", p.formatFieldError(fieldErr), + ), + ) + } + + // from here you can create your own error messages in whatever language you wish + return fmt.Errorf( + "invalid config file %v:\n%v\nPlease make sure the config file is properly formatted", + configPath, + strings.Join(errors, "\n"), + ) +} + +func (p *Provider) formatFieldError(fieldErr validator.FieldError) string { + namespace := strings.TrimLeft(fieldErr.Namespace(), "Config.") + + switch fieldErr.Tag() { + case "required": + return fmt.Sprintf( + "\"%v\"field is required, \"%v\" provided", + namespace, + fieldErr.Value(), + ) + case "min": + if fieldErr.Kind() == reflect.Map || fieldErr.Kind() == reflect.Slice { + return fmt.Sprintf("\"%v\" field must have at least %s element(s)", namespace, fieldErr.Param()) + } + + return fmt.Sprintf("\"%v\" field must have minimum value: %q", namespace, fieldErr.Param()) + default: + return fmt.Sprintf( + "\"%v\"field: %v", + namespace, + fieldErr.Tag(), + ) + } +} + func (p *Provider) Get() *Config { return p.Config } +func (p *Provider) GetStr() string { + loadedConfig, _ := yaml.Marshal(p.Config) + + return string(loadedConfig) +} + func (p *Provider) Start() { } diff --git a/pkg/config/provider_test.go b/pkg/config/provider_test.go index 05f246fb..757c1207 100644 --- a/pkg/config/provider_test.go +++ b/pkg/config/provider_test.go @@ -36,6 +36,27 @@ func TestConfigProvider_ValidConfigLoaded(t *testing.T) { require.Len(t, models, 1) } +func TestConfigProvider_InvalidConfigLoaded(t *testing.T) { + tests := []struct { + name string + configFile string + }{ + {"empty telemetry", "./testdata/provider.telnil.yaml"}, + {"empty logging", "./testdata/provider.loggingnil.yaml"}, + {"no lang routers", "./testdata/provider.nolangrouters.yaml"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + configProvider := NewProvider() + _, err := configProvider.Load(tt.configFile) + + require.Error(t, err) + require.ErrorContains(t, err, "invalid config file") + }) + } +} + func TestConfigProvider_NoProvider(t *testing.T) { configProvider := NewProvider() _, err := configProvider.Load("./testdata/provider.nomodelprovider.yaml") diff --git a/pkg/config/testdata/provider.fullconfig.yaml b/pkg/config/testdata/provider.fullconfig.yaml index 3d960918..e83cfd5d 100644 --- a/pkg/config/testdata/provider.fullconfig.yaml +++ b/pkg/config/testdata/provider.fullconfig.yaml @@ -1,6 +1,6 @@ telemetry: logging: - level: INFO # DEBUG, INFO, WARNING, ERROR, FATAL + level: info # debug, info, warning, error, fatal encoding: json # console, json routers: diff --git a/pkg/config/testdata/provider.loggingnil.yaml b/pkg/config/testdata/provider.loggingnil.yaml new file mode 100644 index 00000000..eab73ea6 --- /dev/null +++ b/pkg/config/testdata/provider.loggingnil.yaml @@ -0,0 +1,15 @@ +telemetry: + logging: + +routers: + language: + - id: simplerouter + strategy: priority + models: + - id: openai-boring + openai: + model: gpt-3.5-turbo + api_key: "ABSC@124" + default_params: + temperature: 0 + diff --git a/pkg/config/testdata/provider.nolangrouters.yaml b/pkg/config/testdata/provider.nolangrouters.yaml new file mode 100644 index 00000000..c70334f0 --- /dev/null +++ b/pkg/config/testdata/provider.nolangrouters.yaml @@ -0,0 +1,7 @@ +telemetry: + logging: + level: info # debug, info, warning, error, fatal + encoding: json # console, json + +routers: + language: [] diff --git a/pkg/config/testdata/provider.telnil.yaml b/pkg/config/testdata/provider.telnil.yaml new file mode 100644 index 00000000..0cf54c72 --- /dev/null +++ b/pkg/config/testdata/provider.telnil.yaml @@ -0,0 +1,14 @@ +telemetry: + +routers: + language: + - id: simplerouter + strategy: priority + models: + - id: openai-boring + openai: + model: gpt-3.5-turbo + api_key: "ABSC@124" + default_params: + temperature: 0 + diff --git a/pkg/gateway.go b/pkg/gateway.go index 0df1f368..4c11a565 100644 --- a/pkg/gateway.go +++ b/pkg/gateway.go @@ -41,6 +41,9 @@ func NewGateway(configProvider *config.Provider) (*Gateway, error) { return nil, err } + tel.Logger.Info("🐦Glide is starting up", zap.String("version", FullVersion)) + tel.Logger.Debug("config loaded successfully:\n" + configProvider.GetStr()) + routerManager, err := routers.NewManager(&cfg.Routers, tel) if err != nil { return nil, err diff --git a/pkg/providers/azureopenai/client.go b/pkg/providers/azureopenai/client.go index 03cce2f2..cd05ba90 100644 --- a/pkg/providers/azureopenai/client.go +++ b/pkg/providers/azureopenai/client.go @@ -30,9 +30,12 @@ type Client struct { // NewClient creates a new Azure OpenAI client for the OpenAI API. func NewClient(providerConfig *Config, clientConfig *clients.ClientConfig, tel *telemetry.Telemetry) (*Client, error) { - chatURL := fmt.Sprintf("%s/openai/deployments/%s/chat/completions?api-version=%s", providerConfig.BaseURL, providerConfig.Model, providerConfig.APIVersion) - - fmt.Println("chatURL", chatURL) + chatURL := fmt.Sprintf( + "%s/openai/deployments/%s/chat/completions?api-version=%s", + providerConfig.BaseURL, + providerConfig.Model, + providerConfig.APIVersion, + ) c := &Client{ baseURL: providerConfig.BaseURL, diff --git a/pkg/providers/config.go b/pkg/providers/config.go index 957cfd66..07691e91 100644 --- a/pkg/providers/config.go +++ b/pkg/providers/config.go @@ -21,20 +21,18 @@ import ( var ErrProviderNotFound = errors.New("provider not found") type LangModelConfig struct { - ID string `yaml:"id" json:"id" validate:"required"` // Model instance ID (unique in scope of the router) - Enabled bool `yaml:"enabled" json:"enabled"` // Is the model enabled? + ID string `yaml:"id" json:"id" validate:"required"` // Model instance ID (unique in scope of the router) + Enabled bool `yaml:"enabled" json:"enabled" validate:"required"` // Is the model enabled? ErrorBudget *health.ErrorBudget `yaml:"error_budget" json:"error_budget" swaggertype:"primitive,string"` Latency *latency.Config `yaml:"latency" json:"latency"` Weight int `yaml:"weight" json:"weight"` Client *clients.ClientConfig `yaml:"client" json:"client"` - OpenAI *openai.Config `yaml:"openai" json:"openai"` - AzureOpenAI *azureopenai.Config `yaml:"azureopenai" json:"azureopenai"` - Cohere *cohere.Config `yaml:"cohere" json:"cohere"` - OctoML *octoml.Config `yaml:"octoml" json:"octoml"` - Anthropic *anthropic.Config `yaml:"anthropic" json:"anthropic"` // Add other providers like - // Cohere *cohere.Config - // Anthropic *anthropic.Config + OpenAI *openai.Config `yaml:"openai,omitempty" json:"openai,omitempty"` + AzureOpenAI *azureopenai.Config `yaml:"azureopenai,omitempty" json:"azureopenai,omitempty"` + Cohere *cohere.Config `yaml:"cohere,omitempty" json:"cohere,omitempty"` + OctoML *octoml.Config `yaml:"octoml,omitempty" json:"octoml,omitempty"` + Anthropic *anthropic.Config `yaml:"anthropic,omitempty" json:"anthropic,omitempty"` } func DefaultLangModelConfig() *LangModelConfig { diff --git a/pkg/routers/config.go b/pkg/routers/config.go index 3dc12c7e..c1c4fbf7 100644 --- a/pkg/routers/config.go +++ b/pkg/routers/config.go @@ -12,15 +12,22 @@ import ( ) type Config struct { - LanguageRouters []LangRouterConfig `yaml:"language"` // the list of language routers + LanguageRouters []LangRouterConfig `yaml:"language" validate:"required,min=1"` // the list of language routers } func (c *Config) BuildLangRouters(tel *telemetry.Telemetry) ([]*LangRouter, error) { + seenIDs := make(map[string]bool, len(c.LanguageRouters)) routers := make([]*LangRouter, 0, len(c.LanguageRouters)) var errs error for idx, routerConfig := range c.LanguageRouters { + if _, ok := seenIDs[routerConfig.ID]; ok { + return nil, fmt.Errorf("ID \"%v\" is specified for more than one router while each ID should be unique", routerConfig.ID) + } + + seenIDs[routerConfig.ID] = true + if !routerConfig.Enabled { tel.Logger.Info("router is disabled, skipping", zap.String("routerID", routerConfig.ID)) continue @@ -48,20 +55,31 @@ func (c *Config) BuildLangRouters(tel *telemetry.Telemetry) ([]*LangRouter, erro // TODO: Had to keep RoutingStrategy because of https://github.com/swaggo/swag/issues/1738 // LangRouterConfig type LangRouterConfig struct { - ID string `yaml:"id" json:"routers" validate:"required"` // Unique router ID - Enabled bool `yaml:"enabled" json:"enabled"` // Is router enabled? - Retry *retry.ExpRetryConfig `yaml:"retry" json:"retry"` // retry when no healthy model is available to router - RoutingStrategy routing.Strategy `yaml:"strategy" json:"strategy" swaggertype:"primitive,string"` // strategy on picking the next model to serve the request - Models []providers.LangModelConfig `yaml:"models" json:"models" validate:"required"` // the list of models that could handle requests + ID string `yaml:"id" json:"routers" validate:"required"` // Unique router ID + Enabled bool `yaml:"enabled" json:"enabled" validate:"required"` // Is router enabled? + Retry *retry.ExpRetryConfig `yaml:"retry" json:"retry" validate:"required"` // retry when no healthy model is available to router + RoutingStrategy routing.Strategy `yaml:"strategy" json:"strategy" swaggertype:"primitive,string" validate:"required"` // strategy on picking the next model to serve the request + Models []providers.LangModelConfig `yaml:"models" json:"models" validate:"required,min=1"` // the list of models that could handle requests } // BuildModels creates LanguageModel slice out of the given config func (c *LangRouterConfig) BuildModels(tel *telemetry.Telemetry) ([]providers.LanguageModel, error) { var errs error + seenIDs := make(map[string]bool, len(c.Models)) models := make([]providers.LanguageModel, 0, len(c.Models)) for _, modelConfig := range c.Models { + if _, ok := seenIDs[modelConfig.ID]; ok { + return nil, fmt.Errorf( + "ID \"%v\" is specified for more than one model in router \"%v\", while it should be unique in scope of that pool", + modelConfig.ID, + c.ID, + ) + } + + seenIDs[modelConfig.ID] = true + if !modelConfig.Enabled { tel.Logger.Info( "model is disabled, skipping", @@ -91,6 +109,19 @@ func (c *LangRouterConfig) BuildModels(tel *telemetry.Telemetry) ([]providers.La return nil, errs } + if len(models) == 0 { + return nil, fmt.Errorf("router \"%v\" must have at least one active model, zero defined", c.ID) + } + + if len(models) == 1 { + tel.Logger.Warn( + "router has only one active model defined. "+ + "This is not recommended for production setups. "+ + "Define at least a few models to leverage resiliency logic Glide provides", + zap.String("router", c.ID), + ) + } + return models, nil } diff --git a/pkg/routers/config_test.go b/pkg/routers/config_test.go index c7270b67..c24cdea1 100644 --- a/pkg/routers/config_test.go +++ b/pkg/routers/config_test.go @@ -39,7 +39,7 @@ func TestRouterConfig_BuildModels(t *testing.T) { }, }, { - ID: "first_router", + ID: "second_router", Enabled: true, RoutingStrategy: routing.LeastLatency, Retry: retry.DefaultExpRetryConfig(), @@ -69,3 +69,115 @@ func TestRouterConfig_BuildModels(t *testing.T) { require.Len(t, routers[1].models, 1) require.IsType(t, routers[1].routing, &routing.LeastLatencyRouting{}) } + +func TestRouterConfig_InvalidSetups(t *testing.T) { + defaultParams := openai.DefaultParams() + + tests := []struct { + name string + config Config + }{ + { + "duplicated router IDs", + Config{ + LanguageRouters: []LangRouterConfig{ + { + ID: "first_router", + Enabled: true, + RoutingStrategy: routing.Priority, + Retry: retry.DefaultExpRetryConfig(), + Models: []providers.LangModelConfig{ + { + ID: "first_model", + Enabled: true, + Client: clients.DefaultClientConfig(), + ErrorBudget: health.DefaultErrorBudget(), + Latency: latency.DefaultConfig(), + OpenAI: &openai.Config{ + APIKey: "ABC", + DefaultParams: &defaultParams, + }, + }, + }, + }, + { + ID: "first_router", + Enabled: true, + RoutingStrategy: routing.LeastLatency, + Retry: retry.DefaultExpRetryConfig(), + Models: []providers.LangModelConfig{ + { + ID: "first_model", + Enabled: true, + Client: clients.DefaultClientConfig(), + ErrorBudget: health.DefaultErrorBudget(), + Latency: latency.DefaultConfig(), + OpenAI: &openai.Config{ + APIKey: "ABC", + DefaultParams: &defaultParams, + }, + }, + }, + }, + }, + }, + }, + { + "duplicated model IDs", + Config{ + LanguageRouters: []LangRouterConfig{ + { + ID: "first_router", + Enabled: true, + RoutingStrategy: routing.Priority, + Retry: retry.DefaultExpRetryConfig(), + Models: []providers.LangModelConfig{ + { + ID: "first_model", + Enabled: true, + Client: clients.DefaultClientConfig(), + ErrorBudget: health.DefaultErrorBudget(), + Latency: latency.DefaultConfig(), + OpenAI: &openai.Config{ + APIKey: "ABC", + DefaultParams: &defaultParams, + }, + }, + { + ID: "first_model", + Enabled: true, + Client: clients.DefaultClientConfig(), + ErrorBudget: health.DefaultErrorBudget(), + Latency: latency.DefaultConfig(), + OpenAI: &openai.Config{ + APIKey: "ABC", + DefaultParams: &defaultParams, + }, + }, + }, + }, + }, + }, + }, + { + "no models", + Config{ + LanguageRouters: []LangRouterConfig{ + { + ID: "first_router", + Enabled: true, + RoutingStrategy: routing.Priority, + Retry: retry.DefaultExpRetryConfig(), + Models: []providers.LangModelConfig{}, + }, + }, + }, + }, + } + + for _, test := range tests { + _, err := test.config.BuildLangRouters(telemetry.NewTelemetryMock()) + + require.Error(t, err) + } +} diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index e1c99428..868dfaeb 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -3,7 +3,7 @@ package telemetry import "go.uber.org/zap" type Config struct { - LogConfig *LogConfig `yaml:"logging"` + LogConfig *LogConfig `yaml:"logging" validate:"required"` // TODO: add OTEL config }