diff --git a/.drone.sh b/.drone.sh new file mode 100644 index 00000000..afd8f2f2 --- /dev/null +++ b/.drone.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e +set -x + +go build \ + -ldflags '-extldflags "-static"' \ + -ldflags "-X main.version=${DRONE_TAG=latest}" \ + -ldflags "-X main.commit=${DRONE_COMMIT_SHA}" \ + -o release/linux/arm64/drone-autoscaler \ + github.com/drone/autoscaler/cmd/drone-autoscaler diff --git a/.drone.yml b/.drone.yml index c116f50d..082218ff 100644 --- a/.drone.yml +++ b/.drone.yml @@ -13,12 +13,7 @@ pipeline: build: image: golang - commands: - - | - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \ - -ldflags "-X main.version=${DRONE_TAG} -X main.commit=${DRONE_COMMIT_SHA}" \ - -o release/linux/arm64/drone-autoscaler \ - github.com/drone/autoscaler/cmd/drone-autoscaler + commands: sh .drone.sh publish: image: plugins/docker @@ -26,4 +21,5 @@ pipeline: auto_tag: true secrets: [ docker_username, docker_password ] when: + branch: master event: [ push, tag ] diff --git a/.gitignore b/.gitignore index 4d6256db..5fe0227d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ NOTES.md release vendor -.*.toml +*.sqlite +*.sqlite3 *.bak *.out *.db diff --git a/Dockerfile b/Dockerfile index a4689485..d9bc9a59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ FROM drone/ca-certs +EXPOSE 8080 80 443 +VOLUME /data -ENV GODEBUG=netdns=go -ENV XDG_CACHE_HOME /var/lib/autoscaler -ENV DRONE_DATABASE_PATH /var/lib/autoscaler/snapshot.db +ENV GODEBUG netdns=go +ENV XDG_CACHE_HOME /data +ENV DATABASE_DRIVER sqlite3 +ENV DATABASE_DATASOURCE /data/database.sqlite?cache=shared&mode=rwc&_busy_timeout=9999999 ADD release/linux/arm64/drone-autoscaler /bin/ - -EXPOSE 8080 80 443 - ENTRYPOINT ["/bin/drone-autoscaler"] diff --git a/Gopkg.lock b/Gopkg.lock index 53929aeb..ecf44342 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -46,12 +46,6 @@ packages = ["."] revision = "307046097ee9f07094bb547c5d96d86d759054a6" -[[projects]] - name = "github.com/boltdb/bolt" - packages = ["."] - revision = "2f1ce7a837dcb8da3ec595b1dac9d0632f0f99e8" - version = "v1.3.1" - [[projects]] branch = "master" name = "github.com/dchest/uniuri" @@ -70,8 +64,7 @@ [[projects]] name = "github.com/drone/drone-go" packages = ["drone"] - revision = "3a20536622c5e513dea26c58f1e997cb4ab4dbc5" - version = "v0.8.4" + revision = "5b15e044e3275274bc54fc711d88ebf73e2061eb" [[projects]] branch = "master" @@ -97,6 +90,12 @@ revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a" version = "v1.32.0" +[[projects]] + name = "github.com/go-sql-driver/mysql" + packages = ["."] + revision = "a0583e0143b1624142adab07e0e97fe106d99561" + version = "v1.3" + [[projects]] name = "github.com/golang/mock" packages = ["gomock"] @@ -121,11 +120,29 @@ revision = "26775f23aa9da4cbd47bc2600a57ac07bbf3bd20" version = "v1.0.7" +[[projects]] + name = "github.com/hetznercloud/hcloud-go" + packages = [ + "hcloud", + "hcloud/schema" + ] + revision = "b37849e439d7a02b07d495f65e7b797bec30eeae" + version = "v1.4.0" + [[projects]] name = "github.com/jmespath/go-jmespath" packages = ["."] revision = "0b12d6b5" +[[projects]] + branch = "master" + name = "github.com/jmoiron/sqlx" + packages = [ + ".", + "reflectx" + ] + revision = "cf35089a197953c69420c8d0cecda90809764b1d" + [[projects]] name = "github.com/joho/godotenv" packages = [ @@ -153,6 +170,12 @@ packages = ["."] revision = "7cafcd837844e784b526369c9bce262804aebc60" +[[projects]] + name = "github.com/mattn/go-sqlite3" + packages = ["."] + revision = "6c771bb9887719704b210e87e934f08be014bdb1" + version = "v1.6.0" + [[projects]] name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] @@ -269,12 +292,6 @@ packages = ["errgroup"] revision = "fd80eb99c8f653c847d294a001bdf2a3a6f768f5" -[[projects]] - branch = "master" - name = "golang.org/x/sys" - packages = ["unix"] - revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" - [[projects]] name = "google.golang.org/appengine" packages = [ @@ -292,6 +309,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "db092cb676d53dc31994a55efa21471b715ea5bc9c8840073e7b4b8c9db0d919" + inputs-digest = "bab8207df652da10439a983a41c736b30690ebb9f973084a01e35c755438d9a8" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index ff3cd21b..59f80e20 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -29,10 +29,6 @@ branch = "master" name = "github.com/bluele/slack" -[[constraint]] - name = "github.com/boltdb/bolt" - version = "1.3.1" - [[constraint]] branch = "master" name = "github.com/dchest/uniuri" @@ -43,7 +39,7 @@ [[constraint]] name = "github.com/drone/drone-go" - version = "0.8.4" + revision = "5b15e044e3275274bc54fc711d88ebf73e2061eb" [[constraint]] branch = "master" @@ -100,3 +96,7 @@ [prune] go-tests = true unused-packages = true + +[[constraint]] + name = "github.com/hetznercloud/hcloud-go" + version = "1.3.1" diff --git a/LICENSE b/LICENSE index 4e39c86a..21d9bff0 100644 --- a/LICENSE +++ b/LICENSE @@ -9,7 +9,7 @@ Additional Use Grant: Usage of the software is free for entities with both: of domicile); and (b) less than (USD) $5 million in aggregate debt and equity funding. -Change Date: 2021-01-01 +Change Date: 2022-03-01 (yyyy-MM-dd) Change License: BSD-3-Clause diff --git a/cmd/drone-autoscaler/main.go b/cmd/drone-autoscaler/main.go index 40e69ce2..901db47c 100644 --- a/cmd/drone-autoscaler/main.go +++ b/cmd/drone-autoscaler/main.go @@ -14,8 +14,9 @@ import ( "github.com/drone/autoscaler" "github.com/drone/autoscaler/config" "github.com/drone/autoscaler/drivers/digitalocean" + "github.com/drone/autoscaler/drivers/hetznercloud" + "github.com/drone/autoscaler/engine" "github.com/drone/autoscaler/metrics" - "github.com/drone/autoscaler/scaler" "github.com/drone/autoscaler/server" "github.com/drone/autoscaler/slack" "github.com/drone/autoscaler/store" @@ -30,7 +31,9 @@ import ( "golang.org/x/oauth2" "golang.org/x/sync/errgroup" + _ "github.com/go-sql-driver/mysql" _ "github.com/joho/godotenv/autoload" + _ "github.com/mattn/go-sqlite3" ) var ( @@ -41,8 +44,6 @@ var ( func main() { conf := config.MustLoad() - metrics.MinPool(conf) - metrics.MaxPool(conf) setupLogging(conf) provider, err := setupProvider(conf) @@ -55,27 +56,30 @@ func main() { provider = metrics.ServerCreate(provider) provider = metrics.ServerDelete(provider) + db, err := store.Connect(conf.Database.Driver, conf.Database.Datasource) + if err != nil { + log.Fatal().Err(err). + Msg("Cannot establish database connection") + } + + servers := store.NewServerStore(db) // instruments the provider with slack notifications // instance creation and termination events. if conf.Slack.Webhook != "" { - provider = slack.New(conf, provider) + servers = slack.New(conf, servers) } - - db := store.Must(conf.Database.Path) - servers := store.NewServerStore(db) - // instruments the store with prometheus metrics. servers = metrics.ServerCount(servers) defer db.Close() client := setupClient(conf) - ascaler := &scaler.Scaler{ - Client: client, - Config: conf, - Servers: servers, - Provider: provider, - } + enginex := engine.New( + client, + conf, + servers, + provider, + ) r := chi.NewRouter() r.Use(hlog.NewHandler(log.Logger)) @@ -87,17 +91,17 @@ func main() { r.Get("/metrics", server.HandleMetrics(conf.Prometheus.Token)) r.Get("/version", server.HandleVersion(source, version, commit)) r.Get("/healthz", server.HandleHealthz()) - r.Get("/varz", server.HandleVarz(ascaler)) + r.Get("/varz", server.HandleVarz(enginex)) r.Route("/api", func(r chi.Router) { r.Use(server.CheckDrone(conf)) - r.Post("/pause", server.HandleScalerPause(ascaler)) - r.Post("/resume", server.HandleScalerResume(ascaler)) + r.Post("/pause", server.HandleEnginePause(enginex)) + r.Post("/resume", server.HandleEngineResume(enginex)) r.Get("/queue", server.HandleQueueList(client)) r.Get("/servers", server.HandleServerList(servers)) - r.Post("/servers", server.HandleServerCreate(servers, provider, conf)) + r.Post("/servers", server.HandleServerCreate(servers, conf)) r.Get("/servers/{name}", server.HandleServerFind(servers)) - r.Delete("/servers/{name}", server.HandleServerDelete(servers, provider)) + r.Delete("/servers/{name}", server.HandleServerDelete(servers)) }) // @@ -134,7 +138,8 @@ func main() { // g.Go(func() error { - return scaler.Start(ctx, ascaler, conf.Interval) + enginex.Start(ctx) + return nil }) if err := g.Wait(); err != nil { @@ -185,6 +190,8 @@ func setupProvider(c config.Config) (autoscaler.Provider, error) { switch { case c.DigitalOcean.Token != "": return digitalocean.FromConfig(c) + case c.HetznerCloud.Token != "": + return hetznercloud.FromConfig(c) default: return nil, errors.New("missing provider configuration") } diff --git a/config/config.go b/config/config.go index 3c3f69cb..0e0d20db 100644 --- a/config/config.go +++ b/config/config.go @@ -56,7 +56,8 @@ type ( } Database struct { - Path string `default:"snapshot.db"` + Driver string `default:"sqlite3"` + Datasource string `default:"database.sqlite?cache=shared&mode=rwc&_busy_timeout=9999999"` } Amazon struct { @@ -93,5 +94,15 @@ type ( Project string Tags []string } + + HetznerCloud struct { + Token string + Image string `default:"ubuntu-16.04"` + Datacenter string `default:"nbg1-dc3"` + SSHKey string `default:"/root/.ssh/id_rsa"` + SSHKeyID int `envconfig:"DRONE_HETZNERCLOUD_SSHKEY_ID"` + ServerType string `default:"cx11" envconfig:"DRONE_HETZNERCLOUD_TYPE"` + CloudConfig string + } } ) diff --git a/config/load_test.go b/config/load_test.go index 5c0aa391..eff5ec04 100644 --- a/config/load_test.go +++ b/config/load_test.go @@ -34,8 +34,11 @@ func TestDefaults(t *testing.T) { if got, want := conf.HTTP.Port, ":8080"; got != want { t.Errorf("Want default DRONE_HTTP_PORT of %s, got %s", want, got) } - if got, want := conf.Database.Path, "snapshot.db"; got != want { - t.Errorf("Want default DRONE_DATABASE_PATH of %s, got %s", want, got) + if got, want := conf.Database.Driver, "sqlite3"; got != want { + t.Errorf("Want default DRONE_DATABASE_DRIVER of %s, got %s", want, got) + } + if got, want := conf.Database.Datasource, "database.sqlite?cache=shared&mode=rwc&_busy_timeout=9999999"; got != want { + t.Errorf("Want default DRONE_DATABASE_DATASOURCE of %s, got %s", want, got) } // // Digital Ocean @@ -64,56 +67,74 @@ func TestDefaults(t *testing.T) { if got, want := conf.Google.DiskType, "pd-standard"; got != want { t.Errorf("Want default DRONE_GOOGLE_DISK_TYPE of %s, got %s", want, got) } + // + // Hetzner Cloud + if got, want := conf.HetznerCloud.Image, "ubuntu-16.04"; got != want { + t.Errorf("Want default DRONE_HETZNERCLOUD_IMAGE of %s, got %s", want, got) + } + if got, want := conf.HetznerCloud.Datacenter, "nbg1-dc3"; got != want { + t.Errorf("Want default DRONE_HETZNERCLOUD_DATACENTER of %s, got %s", want, got) + } + if got, want := conf.HetznerCloud.ServerType, "cx11"; got != want { + t.Errorf("Want default DRONE_HETZNERCLOUD_SERVER_TYPE of %s, got %s", want, got) + } } func TestLoad(t *testing.T) { environ := map[string]string{ - "DRONE_INTERVAL": "1m", - "DRONE_SLACK_WEBHOOK": "https://hooks.slack.com/services/XXX/YYY/ZZZ", - "DRONE_LOGS_DEBUG": "true", - "DRONE_LOGS_COLOR": "true", - "DRONE_LOGS_PRETTY": "true", - "DRONE_POOL_MIN_AGE": "1h", - "DRONE_POOL_MIN": "1", - "DRONE_POOL_MAX": "5", - "DRONE_SERVER_HOST": "drone.company.com", - "DRONE_SERVER_PROTO": "http", - "DRONE_SERVER_TOKEN": "633eb230f5", - "DRONE_HTTP_HOST": "autoscaler.drone.company.com", - "DRONE_HTTP_PORT": "633eb230f5", - "DRONE_AGENT_HOST": "drone.company.com:9000", - "DRONE_AGENT_TOKEN": "f5064039f5", - "DRONE_AGENT_IMAGE": "drone/agent:0.8", - "DRONE_AGENT_CONCURRENCY": "2", - "DRONE_TLS_AUTOCERT": "true", - "DRONE_TLS_CERT": "/path/to/cert.crt", - "DRONE_TLS_KEY": "/path/to/cert.key", - "DRONE_PROMETHEUS_TOKEN": "b359e05e8", - "DRONE_DATABASE_PATH": "/path/to/database.db", - "DRONE_DIGITALOCEAN_TOKEN": "2573633eb", - "DRONE_DIGITALOCEAN_IMAGE": "docker-16-04", - "DRONE_DIGITALOCEAN_REGION": "ncy1", - "DRONE_DIGITALOCEAN_SSHKEY": "/path/to/ssh/key", - "DRONE_DIGITALOCEAN_SIZE": "s-1vcpu-1gb", - "DRONE_DIGITALOCEAN_IPV6": "true", - "DRONE_DIGITALOCEAN_TAGS": "drone,agent,prod", - "DRONE_GOOGLE_ZONE": "us-central1-b", - "DRONE_GOOGLE_MACHINE_TYPE": "f1-micro", - "DRONE_GOOGLE_MACHINE_IMAGE": "ubuntu-1510-wily-v20151114", - "DRONE_GOOGLE_DISK_TYPE": "pd-standard", - "DRONE_GOOGLE_NETWORK": "default", - "DRONE_GOOGLE_SUBNETWORK": "", - "DRONE_GOOGLE_PREEMPTIBLE": "true", - "DRONE_GOOGLE_SCOPES": "devstorage.read_only", - "DRONE_GOOGLE_DISK_SIZE": "10", - "DRONE_GOOGLE_PROJECT": "project-foo", - "DRONE_GOOGLE_TAGS": "drone,agent,prod", - "DRONE_AMAZON_INSTANCE": "t2.medium", - "DRONE_AMAZON_REGION": "us-east-2", - "DRONE_AMAZON_SSHKEY": "/path/to/ssh/key", - "DRONE_AMAZON_SSHKEY_NAME": "id_rsa", - "DRONE_AMAZON_SUBNET_ID": "subnet-0b32177f", - "DRONE_AMAZON_SECURITY_GROUP": "sg-770eabe1", + "DRONE_INTERVAL": "1m", + "DRONE_SLACK_WEBHOOK": "https://hooks.slack.com/services/XXX/YYY/ZZZ", + "DRONE_LOGS_DEBUG": "true", + "DRONE_LOGS_COLOR": "true", + "DRONE_LOGS_PRETTY": "true", + "DRONE_POOL_MIN_AGE": "1h", + "DRONE_POOL_MIN": "1", + "DRONE_POOL_MAX": "5", + "DRONE_SERVER_HOST": "drone.company.com", + "DRONE_SERVER_PROTO": "http", + "DRONE_SERVER_TOKEN": "633eb230f5", + "DRONE_HTTP_HOST": "autoscaler.drone.company.com", + "DRONE_HTTP_PORT": "633eb230f5", + "DRONE_AGENT_HOST": "drone.company.com:9000", + "DRONE_AGENT_TOKEN": "f5064039f5", + "DRONE_AGENT_IMAGE": "drone/agent:0.8", + "DRONE_AGENT_CONCURRENCY": "2", + "DRONE_TLS_AUTOCERT": "true", + "DRONE_TLS_CERT": "/path/to/cert.crt", + "DRONE_TLS_KEY": "/path/to/cert.key", + "DRONE_PROMETHEUS_TOKEN": "b359e05e8", + "DRONE_DATABASE_DRIVER": "mysql", + "DRONE_DATABASE_DATASOURCE": "user:password@/dbname", + "DRONE_DIGITALOCEAN_TOKEN": "2573633eb", + "DRONE_DIGITALOCEAN_IMAGE": "docker-16-04", + "DRONE_DIGITALOCEAN_REGION": "ncy1", + "DRONE_DIGITALOCEAN_SSHKEY": "/path/to/ssh/key", + "DRONE_DIGITALOCEAN_SIZE": "s-1vcpu-1gb", + "DRONE_DIGITALOCEAN_IPV6": "true", + "DRONE_DIGITALOCEAN_TAGS": "drone,agent,prod", + "DRONE_GOOGLE_ZONE": "us-central1-b", + "DRONE_GOOGLE_MACHINE_TYPE": "f1-micro", + "DRONE_GOOGLE_MACHINE_IMAGE": "ubuntu-1510-wily-v20151114", + "DRONE_GOOGLE_DISK_TYPE": "pd-standard", + "DRONE_GOOGLE_NETWORK": "default", + "DRONE_GOOGLE_SUBNETWORK": "", + "DRONE_GOOGLE_PREEMPTIBLE": "true", + "DRONE_GOOGLE_SCOPES": "devstorage.read_only", + "DRONE_GOOGLE_DISK_SIZE": "10", + "DRONE_GOOGLE_PROJECT": "project-foo", + "DRONE_GOOGLE_TAGS": "drone,agent,prod", + "DRONE_AMAZON_INSTANCE": "t2.medium", + "DRONE_AMAZON_REGION": "us-east-2", + "DRONE_AMAZON_SSHKEY": "/path/to/ssh/key", + "DRONE_AMAZON_SSHKEY_NAME": "id_rsa", + "DRONE_AMAZON_SUBNET_ID": "subnet-0b32177f", + "DRONE_AMAZON_SECURITY_GROUP": "sg-770eabe1", + "DRONE_HETZNERCLOUD_TOKEN": "12345678", + "DRONE_HETZNERCLOUD_IMAGE": "ubuntu-16.04", + "DRONE_HETZNERCLOUD_DATACENTER": "nbg1-dc3", + "DRONE_HETZNERCLOUD_SSHKEY": "/path/to/ssh/key", + "DRONE_HETZNERCLOUD_TYPE": "cx11", + "DRONE_HETZNERCLOUD_SSHKEY_ID": "12345", } defer func() { @@ -177,7 +198,8 @@ var jsonConfig = []byte(`{ "Token": "b359e05e8" }, "Database": { - "Path": "/path/to/database.db" + "Driver": "mysql", + "Datasource": "user:password@/dbname" }, "DigitalOcean": { "Token": "2573633eb", @@ -217,5 +239,13 @@ var jsonConfig = []byte(`{ "agent", "prod" ] + }, + "HetznerCloud": { + "Token": "12345678", + "Image": "ubuntu-16.04", + "Datacenter": "nbg1-dc3", + "SSHKey": "/path/to/ssh/key", + "ServerType": "cx11", + "SSHKeyID": "12345" } }`) diff --git a/drivers/amazon/amazon.go b/drivers/amazon/amazon.go index f09ccd3a..2e8bc256 100644 --- a/drivers/amazon/amazon.go +++ b/drivers/amazon/amazon.go @@ -5,15 +5,10 @@ package amazon import ( - "context" - "time" - - "github.com/dchest/uniuri" "github.com/drone/autoscaler" "github.com/drone/autoscaler/config" "github.com/drone/autoscaler/drivers/internal/scripts" - "github.com/drone/autoscaler/drivers/internal/sshutil" - "github.com/rs/zerolog/log" + "golang.org/x/crypto/ssh" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -31,206 +26,10 @@ const ( // Provider defines the Amazon provider. type Provider struct { - config config.Config -} - -// Create creates the Amazon instance. -func (p *Provider) Create(ctx context.Context, opts *autoscaler.ServerOpts) (*autoscaler.Server, error) { - client := p.getClient() - - signer, err := sshutil.ParsePrivateKey(p.config.Amazon.SSHKey) - if err != nil { - return nil, err - } - - in := &ec2.RunInstancesInput{ - KeyName: aws.String(p.config.Amazon.SSHKeyName), - ImageId: aws.String(defaultImage), - InstanceType: aws.String(p.config.Amazon.Instance), - MinCount: aws.Int64(1), - MaxCount: aws.Int64(1), - NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{ - { - AssociatePublicIpAddress: aws.Bool(true), - DeviceIndex: aws.Int64(0), - SubnetId: aws.String(p.config.Amazon.SubnetID), - Groups: aws.StringSlice(p.config.Amazon.SecurityGroup), - }, - }, - } - - logger := log.Ctx(ctx).With(). - Str("image", *in.ImageId). - Str("type", *in.InstanceType). - Str("name", opts.Name). - Logger() - - logger.Debug(). - Msg("instance create") - - results, err := client.RunInstances(in) - if err != nil { - logger.Error(). - Err(err). - Msg("instance create failed") - return nil, err - } - - instance := results.Instances[0] - - // tag the instance with user-defined tags. - if tags := p.config.Amazon.Tags; tags != nil && len(tags) != 0 { - p.getClient().CreateTags(&ec2.CreateTagsInput{ - Resources: []*string{instance.InstanceId}, - Tags: convertTags(p.config.Amazon.Tags), - }) - } - - server := &autoscaler.Server{ - Provider: autoscaler.ProviderAmazon, - UID: *instance.InstanceId, - Name: opts.Name, - Size: *in.InstanceType, - Region: *instance.Placement.AvailabilityZone, - Image: *in.ImageId, - Capacity: opts.Capacity, - Secret: opts.Secret, - } - - // wait for the server to be available - for { - logger.Debug(). - Str("name", server.Name). - Msg("instance network check") - - desc, err := client.DescribeInstances(&ec2.DescribeInstancesInput{ - InstanceIds: []*string{instance.InstanceId}, - }) - if err != nil { - logger.Error(). - Err(err). - Msg("instance details failed") - return nil, err - } - instance = desc.Reservations[0].Instances[0] - - if instance.PublicIpAddress == nil { - time.Sleep(time.Minute) - continue - } - break - } - - server.Address = *instance.PublicIpAddress - - logger.Debug(). - Str("name", server.Name). - Str("ip", server.Address). - Msg("instance network address assigned") - - // ping the server in a loop until we can successfully - // authenticate. - for { - logger.Debug(). - Str("name", server.Name). - Str("ip", server.Address). - Str("port", "22"). - Str("user", "ubuntu"). - Msg("ping server") - _, err = sshutil.Execute(server.Address, "22", "ubuntu", "whoami", signer) - if err == nil { - break - } else { - time.Sleep(time.Minute) - } - } - - server.Secret = uniuri.New() - server.Created = time.Now().Unix() - server.Updated = time.Now().Unix() + autoscaler.Provider - script, err := scripts.GenerateInstall(p.config, server) - if err != nil { - return server, err - } - - logger.Debug(). - Str("name", server.Name). - Str("ip", server.Address). - Msg("install agent") - - out, err := sshutil.Execute(server.Address, "22", "ubuntu", script, signer) - server.Logs = string(out) - if err != nil { - logger.Error(). - Err(err). - Str("name", server.Name). - Str("ip", server.Address). - Msg("install failed") - return server, err - } - - logger.Debug(). - Str("name", server.Name). - Str("ip", server.Address). - Msg("install complete") - - return server, nil -} - -// Destroy terminates the AWS instance. -func (p *Provider) Destroy(ctx context.Context, server *autoscaler.Server) error { - logger := log.Ctx(ctx).With(). - Str("region", server.Region). - Str("image", server.Image). - Str("size", server.Size). - Str("name", server.Name). - Logger() - - script, err := scripts.GenerateTeardown(p.config) - if err != nil { - return err - } - - signer, err := sshutil.ParsePrivateKey(p.config.Amazon.SSHKey) - if err != nil { - return err - } - - logger.Debug(). - Msg("teardown instance") - - _, err = sshutil.Execute(server.Address, "22", "ubuntu", script, signer) - if err != nil { - logger.Error(). - Err(err). - Msg("teardown failed") - return err - } - - logger.Debug(). - Msg("teardown instance complete") - - logger.Debug(). - Msg("terminate instance") - - input := &ec2.TerminateInstancesInput{ - InstanceIds: []*string{ - aws.String(server.UID), - }, - } - _, err = p.getClient().TerminateInstances(input) - if err != nil { - logger.Error(). - Err(err). - Msg("terminate instance failed") - return err - } - - logger.Debug(). - Msg("instance terminated") - - return nil + config config.Config + signer ssh.Signer } func (p *Provider) getClient() *ec2.EC2 { @@ -243,3 +42,16 @@ func buildClient(conf config.Config) *ec2.EC2 { config = config.WithMaxRetries(maxRetries) return ec2.New(session.New(config)) } + +func (p *Provider) setupScriptOpts(instance *autoscaler.Instance) scripts.SetupOpts { + opts := scripts.SetupOpts{} + opts.Server.Host = p.config.Agent.Host + opts.Server.Secret = p.config.Agent.Token + opts.Agent.Image = p.config.Agent.Image + opts.Agent.Capacity = p.config.Agent.Concurrency + opts.Instance.Addr = instance.Address + opts.Instance.Name = instance.Name + opts.Cadvisor.Disable = false + opts.Cadvisor.Secret = instance.Secret + return opts +} diff --git a/drivers/amazon/create.go b/drivers/amazon/create.go new file mode 100644 index 00000000..94cf5bc2 --- /dev/null +++ b/drivers/amazon/create.go @@ -0,0 +1,178 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package amazon + +import ( + "context" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/dchest/uniuri" + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/drivers/internal/scripts" + "github.com/rs/zerolog/log" +) + +// Create creates the DigitalOcean instance. +func (p *Provider) Create(ctx context.Context, opts autoscaler.InstanceCreateOpts) (*autoscaler.Instance, error) { + client := p.getClient() + + // BEGIN:_PROVIDER_SPECIFIC_CODE + in := &ec2.RunInstancesInput{ + KeyName: aws.String(p.config.Amazon.SSHKeyName), + ImageId: aws.String(defaultImage), + InstanceType: aws.String(p.config.Amazon.Instance), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{ + { + AssociatePublicIpAddress: aws.Bool(true), + DeviceIndex: aws.Int64(0), + SubnetId: aws.String(p.config.Amazon.SubnetID), + Groups: aws.StringSlice(p.config.Amazon.SecurityGroup), + }, + }, + } + // END:_PROVIDER_SPECIFIC_CODE + + logger := log.Ctx(ctx).With(). + Str("region", p.config.Amazon.Region). + Str("image", defaultImage). + Str("size", p.config.Amazon.Instance). + Str("name", opts.Name). + Logger() + + logger.Debug(). + Msg("instance create") + + // BEGIN:_PROVIDER_SPECIFIC_CODE + results, err := client.RunInstances(in) + if err != nil { + logger.Error(). + Err(err). + Msg("instance create failed") + return nil, err + } + + amazonInstance := results.Instances[0] + + instance := &autoscaler.Instance{ + Provider: autoscaler.ProviderAmazon, + ID: *amazonInstance.InstanceId, + Name: opts.Name, + Size: *amazonInstance.InstanceType, + Region: *amazonInstance.Placement.AvailabilityZone, + Image: *amazonInstance.ImageId, + Secret: uniuri.New(), + } + // END:_PROVIDER_SPECIFIC_CODE + + logger.Info(). + Str("name", instance.Name). + Msg("instance create success") + + // poll the digitalocean endpoint for server updates + // and exit when a network address is allocated. + interval := time.Duration(0) +poller: + for { + select { + case <-ctx.Done(): + logger.Debug(). + Str("name", instance.Name). + Msg("instance network deadline exceeded") + + return instance, ctx.Err() + case <-time.After(interval): + interval = time.Minute + + logger.Debug(). + Str("name", instance.Name). + Msg("check instance network") + + // BEGIN:_PROVIDER_SPECIFIC_CODE + desc, err := client.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: []*string{amazonInstance.InstanceId}, + }) + if err != nil { + logger.Error(). + Err(err). + Msg("instance details failed") + return nil, err + } + amazonInstance = desc.Reservations[0].Instances[0] + + if amazonInstance.PublicIpAddress != nil { + instance.Address = *amazonInstance.PublicIpAddress + break poller + } + // END:_PROVIDER_SPECIFIC_CODE + } + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("instance network ready") + + // ping the server in a loop until we can successfully + // authenticate. + interval = time.Duration(0) +pinger: + for { + select { + case <-ctx.Done(): + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Str("port", "22"). + Str("user", "root"). + Msg("ping deadline exceeded") + + return instance, ctx.Err() + case <-time.After(interval): + interval = time.Minute + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Str("port", "22"). + Str("user", "root"). + Msg("ping server") + + err = p.Provider.Ping(ctx, instance) + if err == nil { + break pinger + } + } + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install agent") + + script, err := scripts.GenerateSetup(p.setupScriptOpts(instance)) + if err != nil { + return instance, err + } + + logs, err := p.Provider.Execute(ctx, instance, script) + if err != nil { + logger.Error(). + Err(err). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install failed") + return instance, &autoscaler.InstanceError{Err: err, Logs: logs} + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install complete") + + return instance, nil +} diff --git a/drivers/amazon/create_test.go b/drivers/amazon/create_test.go new file mode 100644 index 00000000..fd5123a7 --- /dev/null +++ b/drivers/amazon/create_test.go @@ -0,0 +1,5 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package amazon diff --git a/drivers/amazon/destroy.go b/drivers/amazon/destroy.go new file mode 100644 index 00000000..65035125 --- /dev/null +++ b/drivers/amazon/destroy.go @@ -0,0 +1,67 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package amazon + +import ( + "context" + + "github.com/drone/autoscaler" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/rs/zerolog/log" +) + +// Destroy destroyes the DigitalOcean instance. +func (p *Provider) Destroy(ctx context.Context, instance *autoscaler.Instance) error { + logger := log.Ctx(ctx).With(). + Str("id", instance.ID). + Str("ip", instance.Address). + Str("name", instance.Name). + Str("zone", instance.Region). + Logger() + + logger.Debug(). + Msg("shutdown agent") + + _, err := p.Provider.Execute(ctx, instance, teardownScript) + if err != nil { + // if we cannot gracefully shutdown the agent we should + // still continue and destroy the instance. I think. + logger.Error(). + Err(err). + Msg("cannot shutdown agent") + + // TODO(bradrydzewski) we should snapshot the error logs + } + + logger.Debug(). + Msg("terminate instsance") + + input := &ec2.TerminateInstancesInput{ + InstanceIds: []*string{ + aws.String(instance.ID), + }, + } + _, err = p.getClient().TerminateInstances(input) + if err != nil { + logger.Error(). + Err(err). + Msg("cannot terminate instance") + return err + } + + logger.Debug(). + Msg("terminated") + + return nil +} + +var teardownScript = ` +set -x; + +sudo docker stop -t 3600 agent +sudo docker ps -a +` diff --git a/drivers/amazon/destroy_test.go b/drivers/amazon/destroy_test.go new file mode 100644 index 00000000..fd5123a7 --- /dev/null +++ b/drivers/amazon/destroy_test.go @@ -0,0 +1,5 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package amazon diff --git a/drivers/digitalocean/create.go b/drivers/digitalocean/create.go new file mode 100644 index 00000000..7f838cf9 --- /dev/null +++ b/drivers/digitalocean/create.go @@ -0,0 +1,180 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package digitalocean + +import ( + "context" + "strconv" + "time" + + "github.com/dchest/uniuri" + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/drivers/internal/scripts" + "github.com/drone/autoscaler/drivers/internal/sshutil" + + "github.com/digitalocean/godo" + "github.com/rs/zerolog/log" +) + +// Create creates the DigitalOcean instance. +func (p *Provider) Create(ctx context.Context, opts autoscaler.InstanceCreateOpts) (*autoscaler.Instance, error) { + req := &godo.DropletCreateRequest{ + Name: opts.Name, + Region: p.config.DigitalOcean.Region, + Size: p.config.DigitalOcean.Size, + IPv6: p.config.DigitalOcean.IPv6, + Tags: p.config.DigitalOcean.Tags, + SSHKeys: []godo.DropletCreateSSHKey{ + {Fingerprint: sshutil.Fingerprint(p.signer)}, + }, + Image: godo.DropletCreateImage{ + Slug: p.config.DigitalOcean.Image, + }, + } + if req.Image.Slug == "" { + req.Image.Slug = "docker-16-04" + } + if req.Size == "" { + req.Size = "s-1vcpu-1gb" + } + if req.Region == "" { + req.Region = "sfo1" + } + + logger := log.Ctx(ctx).With(). + Str("region", req.Region). + Str("image", req.Image.Slug). + Str("size", req.Size). + Str("name", req.Name). + Logger() + + logger.Debug(). + Msg("droplet create") + + client := newClient(ctx, p.config.DigitalOcean.Token) + droplet, _, err := client.Droplets.Create(ctx, req) + if err != nil { + logger.Error(). + Err(err). + Msg("droplet create failed") + return nil, err + } + + instance := &autoscaler.Instance{ + Provider: autoscaler.ProviderDigitalOcean, + ID: strconv.Itoa(droplet.ID), + Name: droplet.Name, + Size: req.Size, + Region: req.Region, + Image: req.Image.Slug, + Secret: uniuri.New(), + } + + logger.Info(). + Str("name", instance.Name). + Msg("droplet create success") + + // poll the digitalocean endpoint for server updates + // and exit when a network address is allocated. + interval := time.Duration(0) +poller: + for { + select { + case <-ctx.Done(): + logger.Debug(). + Str("name", instance.Name). + Msg("droplet network deadline exceeded") + + return instance, ctx.Err() + case <-time.After(interval): + interval = time.Minute + + logger.Debug(). + Str("name", instance.Name). + Msg("check droplet network") + + droplet, _, err = client.Droplets.Get(ctx, droplet.ID) + if err != nil { + logger.Error(). + Err(err). + Msg("droplet details unavailable") + return instance, err + } + + for _, network := range droplet.Networks.V4 { + if network.Type == "public" { + instance.Address = network.IPAddress + } + } + + if instance.Address != "" { + break poller + } + } + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("droplet network ready") + + // ping the server in a loop until we can successfully + // authenticate. + interval = time.Duration(0) +pinger: + for { + select { + case <-ctx.Done(): + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Str("port", "22"). + Str("user", "root"). + Msg("ping deadline exceeded") + + return instance, ctx.Err() + case <-time.After(interval): + interval = time.Minute + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Str("port", "22"). + Str("user", "root"). + Msg("ping server") + + err = p.Provider.Ping(ctx, instance) + if err == nil { + break pinger + } + } + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install agent") + + script, err := scripts.GenerateSetup(p.setupScriptOpts(instance)) + if err != nil { + return instance, err + } + + logs, err := p.Provider.Execute(ctx, instance, script) + if err != nil { + logger.Error(). + Err(err). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install failed") + return instance, &autoscaler.InstanceError{Err: err, Logs: logs} + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install complete") + + return instance, nil +} diff --git a/drivers/digitalocean/create_test.go b/drivers/digitalocean/create_test.go new file mode 100644 index 00000000..48ed091c --- /dev/null +++ b/drivers/digitalocean/create_test.go @@ -0,0 +1,403 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package digitalocean + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + "github.com/digitalocean/godo" + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/autoscaler/mocks" + "github.com/golang/mock/gomock" + + "github.com/h2non/gock" + "golang.org/x/crypto/ssh" +) + +func TestCreate(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Post("/v2/droplets"). + Reply(200). + BodyString(respDropletCreate) + + gock.New("https://api.digitalocean.com"). + Get("/v2/droplets/3164494"). + Reply(200). + BodyString(respDropletDesc) + + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Ping(gomock.Any(), gomock.Any()).Return(nil) + mockProvider.EXPECT().Execute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + instance, err := p.Create(context.TODO(), autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err != nil { + t.Error(err) + } + + t.Run("Attributes", testInstance(instance)) +} + +func TestCreate_CreateError(t *testing.T) { + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Post("/v2/droplets"). + Reply(500) + + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + p := Provider{ + Provider: nil, + config: mockConfig, + signer: mockSigner, + } + + _, err := p.Create(context.TODO(), autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err == nil { + t.Errorf("Expect error returned from digital ocean") + } else if _, ok := err.(*godo.ErrorResponse); !ok { + t.Errorf("Expect ErrorResponse digital ocean") + } +} + +func TestCreate_DescribeError(t *testing.T) { + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Post("/v2/droplets"). + Reply(200). + BodyString(respDropletCreate) + + gock.New("https://api.digitalocean.com"). + Get("/v2/droplets/3164494"). + Reply(500) + + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + p := Provider{ + Provider: nil, + config: mockConfig, + signer: mockSigner, + } + + instance, err := p.Create(context.TODO(), autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err == nil { + t.Errorf("Expect error returned from digital ocean") + } else if _, ok := err.(*godo.ErrorResponse); !ok { + t.Errorf("Expect ErrorResponse digital ocean") + } + + t.Run("Attributes", testInstance(instance)) +} + +func TestCreate_DescribeTimeout(t *testing.T) { + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Post("/v2/droplets"). + Reply(200). + BodyString(respDropletCreate) + + gock.New("https://api.digitalocean.com"). + Get("/v2/droplets/3164494"). + Reply(200). + BodyString(respDropletCreate) // no network data + + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + p := Provider{ + Provider: nil, + config: mockConfig, + signer: mockSigner, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + instance, err := p.Create(ctx, autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err == nil { + t.Errorf("Expected context deadline exceeded, got nil") + } else if err.Error() != "context deadline exceeded" { + t.Errorf("Expected context deadline exceeded, got %s", err) + } + + t.Run("Attributes", testInstance(instance)) +} + +func TestCreate_PingTimeout(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Post("/v2/droplets"). + Reply(200). + BodyString(respDropletCreate) + + gock.New("https://api.digitalocean.com"). + Get("/v2/droplets/3164494"). + ReplyFunc(func(res *gock.Response) { + res.Status(200) + res.BodyString(respDropletDesc) + }) + + mockError := errors.New("oh no") + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Ping(ctx, gomock.Any()).Return(mockError) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + instance, err := p.Create(ctx, autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err == nil { + t.Errorf("Expected context deadline exceeded, got nil") + } else if err.Error() != "context deadline exceeded" { + t.Errorf("Expected context deadline exceeded, got %s", err) + } + + t.Run("Attributes", testInstance(instance)) + t.Run("Address", testInstanceAddress(instance)) +} + +func TestCreate_ExecError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Post("/v2/droplets"). + Reply(200). + BodyString(respDropletCreate) + + gock.New("https://api.digitalocean.com"). + Get("/v2/droplets/3164494"). + Reply(200). + BodyString(respDropletDesc) + + mockContext := context.Background() + mockLogs := []byte("-bash: curl: command not found") + mockError := errors.New("uh oh") + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Ping(mockContext, gomock.Any()).Return(nil) + mockProvider.EXPECT().Execute(mockContext, gomock.Any(), gomock.Any()).Return(mockLogs, mockError) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + instance, err := p.Create(context.TODO(), autoscaler.InstanceCreateOpts{Name: "agent1"}) + if lerr, ok := err.(*autoscaler.InstanceError); !ok { + t.Errorf("Want InstanceError") + } else if err == nil { + t.Errorf("Want InstanceError got nil") + } else if lerr.Err != mockError { + t.Errorf("Want InstanceError to wrap the ssh error") + } else if !bytes.Equal(lerr.Logs, mockLogs) { + t.Errorf("Want InstanceError to include the logs") + } + + t.Run("Attributes", testInstance(instance)) + t.Run("Address", testInstanceAddress(instance)) +} + +func testInstance(instance *autoscaler.Instance) func(t *testing.T) { + return func(t *testing.T) { + if instance == nil { + t.Errorf("Expect non-nil instance even if error") + } + if got, want := instance.ID, "3164494"; got != want { + t.Errorf("Want droplet ID %v, got %v", want, got) + } + if got, want := instance.Image, "docker-16-04"; got != want { + t.Errorf("Want droplet Image %v, got %v", want, got) + } + if got, want := instance.Name, "example.com"; got != want { + t.Errorf("Want droplet Name %v, got %v", want, got) + } + if got, want := instance.Region, "sfo1"; got != want { + t.Errorf("Want droplet Region %v, got %v", want, got) + } + if got, want := instance.Provider, autoscaler.ProviderDigitalOcean; got != want { + t.Errorf("Want droplet Provider %v, got %v", want, got) + } + if instance.Secret == "" { + t.Errorf("Want instance secret populated, got empty") + } + } +} + +func testInstanceAddress(instance *autoscaler.Instance) func(t *testing.T) { + return func(t *testing.T) { + if instance == nil { + t.Errorf("Expect non-nil instance even if error") + } + if got, want := instance.Address, "104.131.186.241"; got != want { + t.Errorf("Want droplet Address %v, got %v", want, got) + } + } +} + +// sample response for POST /v2/droplets +const respDropletCreate = ` +{ + "droplet": { + "id": 3164494, + "name": "example.com", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "locked": true, + "status": "new", + "kernel": { + "id": 2233, + "name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic", + "version": "3.13.0-37-generic" + }, + "created_at": "2014-11-14T16:36:31Z", + "features": [ + "virtio" + ], + "backup_ids": [ + + ], + "snapshot_ids": [ + + ], + "image": { + + }, + "volume_ids": [ + + ], + "size": { + + }, + "size_slug": "s-1vcpu-1gb", + "networks": { + + }, + "region": { + + }, + "tags": [ + "web" + ] + }, + "links": { + "actions": [ + { + "id": 36805096, + "rel": "create", + "href": "https:\/\/api.digitalocean.com\/v2\/actions\/36805096" + } + ] + } +} +` + +// sample response for POST /v2/droplets/:id +const respDropletDesc = ` +{ + "droplet": { + "id": 3164494, + "name": "example.com", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "locked": true, + "status": "new", + "kernel": { + "id": 2233, + "name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic", + "version": "3.13.0-37-generic" + }, + "created_at": "2014-11-14T16:36:31Z", + "features": [ + "virtio" + ], + "backup_ids": [ + + ], + "snapshot_ids": [ + + ], + "image": { + + }, + "volume_ids": [ + + ], + "size": { + + }, + "size_slug": "s-1vcpu-1gb", + "networks": { + "v4": [ + { + "ip_address": "104.131.186.241", + "netmask": "255.255.240.0", + "gateway": "104.131.176.1", + "type": "public" + } + ] + }, + "region": { + + }, + "tags": [ + "web" + ] + }, + "links": { + "actions": [ + { + "id": 36805096, + "rel": "create", + "href": "https:\/\/api.digitalocean.com\/v2\/actions\/36805096" + } + ] + } +} +` diff --git a/drivers/digitalocean/destroy.go b/drivers/digitalocean/destroy.go new file mode 100644 index 00000000..88abd5cc --- /dev/null +++ b/drivers/digitalocean/destroy.go @@ -0,0 +1,67 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package digitalocean + +import ( + "context" + "strconv" + + "github.com/drone/autoscaler" + + "github.com/rs/zerolog/log" +) + +// Destroy destroyes the DigitalOcean instance. +func (p *Provider) Destroy(ctx context.Context, instance *autoscaler.Instance) error { + logger := log.Ctx(ctx).With(). + Str("region", instance.Region). + Str("image", instance.Image). + Str("size", instance.Size). + Str("name", instance.Name). + Logger() + + client := newClient(ctx, p.config.DigitalOcean.Token) + id, err := strconv.Atoi(instance.ID) + if err != nil { + return err + } + + logger.Debug(). + Msg("teardown droplet") + + _, err = p.Provider.Execute(ctx, instance, teardownScript) + if err != nil { + // if we cannot gracefully shutdown the agent we should + // still continue and destroy the droplet. I think. + logger.Error(). + Err(err). + Msg("teardown failed") + + // TODO(bradrydzewski) we should snapshot the error logs + } + + logger.Debug(). + Msg("deleting droplet") + + _, err = client.Droplets.Delete(ctx, id) + if err != nil { + logger.Error(). + Err(err). + Msg("deleting droplet failed") + return err + } + + logger.Debug(). + Msg("droplet deleted") + + return nil +} + +var teardownScript = ` +set -x; + +docker stop -t 3600 agent +docker ps -a +` diff --git a/drivers/digitalocean/destroy_test.go b/drivers/digitalocean/destroy_test.go new file mode 100644 index 00000000..0668c412 --- /dev/null +++ b/drivers/digitalocean/destroy_test.go @@ -0,0 +1,162 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package digitalocean + +import ( + "context" + "errors" + "strconv" + "testing" + + "github.com/digitalocean/godo" + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/autoscaler/mocks" + + "github.com/golang/mock/gomock" + "github.com/h2non/gock" + "golang.org/x/crypto/ssh" +) + +func TestDestroy(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Delete("/v2/droplets/3164494"). + Reply(204) + + mockContext := context.TODO() + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + mockInstance := &autoscaler.Instance{ + ID: "3164494", + } + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Execute(mockContext, mockInstance, gomock.Any()).Return(nil, nil) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + err := p.Destroy(mockContext, mockInstance) + if err != nil { + t.Error(err) + } +} + +func TestDestroyShutdownError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Delete("/v2/droplets/3164494"). + Reply(204) + + mockError := errors.New("oh no") + mockContext := context.TODO() + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + mockInstance := &autoscaler.Instance{ + ID: "3164494", + } + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Execute(mockContext, mockInstance, gomock.Any()).Return(nil, mockError) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + err := p.Destroy(mockContext, mockInstance) + if err != nil { + t.Error(err) + } +} + +func TestDestroyDeleteError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.digitalocean.com"). + Delete("/v2/droplets/3164494"). + Reply(500) + + mockContext := context.TODO() + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + mockInstance := &autoscaler.Instance{ + ID: "3164494", + } + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Execute(mockContext, mockInstance, gomock.Any()).Return(nil, nil) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + err := p.Destroy(mockContext, mockInstance) + if err == nil { + t.Errorf("Expect error returned from digital ocean") + } else if _, ok := err.(*godo.ErrorResponse); !ok { + t.Errorf("Expect ErrorResponse digital ocean") + } +} + +func TestDestroyInvalidInput(t *testing.T) { + i := &autoscaler.Instance{} + p := Provider{} + err := p.Destroy(context.TODO(), i) + if _, ok := err.(*strconv.NumError); !ok { + t.Errorf("Expected invalid or missing ID error") + } +} + +var testkey = []byte(` +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAwz+uIrhrf/C+Ku0EaofJQvPrmkQpeV2Bx3zVWjJ7GGn1k8GQ +bGXhNEsvzV3XAoFEkSD3uc6W5fU0/HxdbLtxFxUy2dg8tO/AmTrVsHAtZBMjS6ha +2ada/Xg/iH2ZvWj2t5XCAcOpotl5SBafZYytfWal2IEQ/qBEm8olZqyo3GGPxCBn +vUspbJt/i4Nzp/0mQmUrwv5EvaJa8G0ILt/AmYrKsFY2VCTt8v4XRJaPYejSyTDt +DddH6HWxIzc5fn0WMD+nCLb+a7Q33RPfZ7Y2CmuG088INEwYNsURYVLGzHwNkT7a +lHmDVH2+0YgU08m9bnEFTkcL1aH/HVIN/tXe7wIDAQABAoIBAF1FL19gr+HHVGDX +JrPpN8inExZ3l0Rl2dg9FwJmeQ05mNnDrsVJieJcRHKbcFm+/M1DbXOyb71cfLpc +gpitliGLu+X6+U0J9vx78Za+j8Btr/+1ZejxnHLXHaqLLYUg/jLG9I25NXEY6Gn6 +fJybLkloXrNlPIQWdY/iail5M5VKo/CtOuyoSNJzN2HShe+uU4CR6js4URK7QiAA +y6dUW4VtkYlZCOATqHIMUAIx5fi/734v5b8ABUHOFNpaLBbUgKyvPS0H4MatjAJf +n4MiEj0A+Fyvv1UiXQJV5uQ5/Z3mv4Zf9dxZ6qHbGEzdGZWB19AClYTwKYfu/odu +IK5ubjECgYEA6uzRiDGxnB/Xfbb19O4WDgGut/qwP9qm+2/z08HFRv8n1VGSXsWY +AYl0VNAYbovOCGHGzhubYWbw0RenYaL/9YPiWa+ealMbDf6YsmU+0pvXuzyFtYI9 +RHIlP81ViiDXcLzu/H7BVvEv4DfHSD9jFkWMlc83TCjQX/kEuSnulUcCgYEA1MOs +bB5V/Sw4dVj/a46zdo9ZK3JkTrNnqdI8nYqlqqSO4L11i2tJWhoh5ueDJ+NLiEIu +Wujdba7I3LzyetGITTREsLShPmM0EIuX0jJpeeTO3ylaUJppiummnsGmqMfVqdik +UYrwlD0ekm5vko7tUZqmEIgVXNA2kGDyNpR93RkCgYAUP997vtTRYUlA09F1kEQk +Zu65ewlQJ7e2+ppoyU4I5Zt4XrSgKKYGk+OMH/fLJ4/V1x+8ylJlXesqCsDpwJQR +hJGxK1sbTRiK50QgNGvq2XYJ9JiN4bEIQlKFolxaMKSBWje7We2uYdG/oO8zggs3 +cz0/+IGKtgXoD93hXATtpwKBgFyPb9Btdh05AqrSd/Pz1dErVbCYCFlQpTV099fV +vHK7Okk9QwjPOM8Q9VS9vQo6UN7LY9061zHjSxD0xkx2IWTs60Ewo8E/aSQVhov0 +UHyt9O2S0O6l7mp3cXw5ZOaiYSqNzBaJalYjLMypbLKGqWnJ7JreiOSi1EoFUvo5 +qXPpAoGAVZCcIes93+9MHbcbhVyHv3X0XulrPQMRcve0M4MarZ7UWAIPrcIKhQC9 +2M+S1L0BJaHZxTQyxCb2XNRKqzQnmuhrT/A+tH8SUpl2ZLlF1KiNn4UsznBrvsaJ +OoKlArIydogC0Ugu2LoKMw8oIkpf2ANTii3dYJN6ulBKE6EYaRk= +-----END RSA PRIVATE KEY----- +`) diff --git a/drivers/digitalocean/digitalocean.go b/drivers/digitalocean/digitalocean.go deleted file mode 100644 index 75d0ee17..00000000 --- a/drivers/digitalocean/digitalocean.go +++ /dev/null @@ -1,247 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package digitalocean - -import ( - "context" - "strconv" - "time" - - "github.com/dchest/uniuri" - "github.com/drone/autoscaler" - "github.com/drone/autoscaler/config" - "github.com/drone/autoscaler/drivers/internal/scripts" - "github.com/drone/autoscaler/drivers/internal/sshutil" - - "github.com/digitalocean/godo" - "github.com/rs/zerolog/log" - "golang.org/x/oauth2" -) - -// Provider defines the DigitalOcean provider. -type Provider struct { - config config.Config -} - -// FromConfig loads the provider from the configuration. -func FromConfig(config config.Config) (autoscaler.Provider, error) { - return &Provider{ - config: config, - }, nil -} - -// Create creates the DigitalOcean instance. -func (p *Provider) Create(ctx context.Context, opts *autoscaler.ServerOpts) (*autoscaler.Server, error) { - signer, err := sshutil.ParsePrivateKey(p.config.DigitalOcean.SSHKey) - if err != nil { - return nil, err - } - - req := &godo.DropletCreateRequest{ - Name: opts.Name, - Region: p.config.DigitalOcean.Region, - Size: p.config.DigitalOcean.Size, - IPv6: p.config.DigitalOcean.IPv6, - Tags: p.config.DigitalOcean.Tags, - SSHKeys: []godo.DropletCreateSSHKey{ - {Fingerprint: sshutil.Fingerprint(signer)}, - }, - Image: godo.DropletCreateImage{ - Slug: p.config.DigitalOcean.Image, - }, - } - - if req.Image.Slug == "" { - req.Image.Slug = "docker-16-04" - } - if req.Size == "" { - req.Size = "s-1vcpu-1gb" - } - if req.Region == "" { - req.Region = "sfo1" - } - - logger := log.Ctx(ctx).With(). - Str("region", req.Region). - Str("image", req.Image.Slug). - Str("size", req.Size). - Str("name", req.Name). - Logger() - - logger.Debug(). - Msg("droplet create") - - client := newClient(ctx, p.config.DigitalOcean.Token) - droplet, _, err := client.Droplets.Create(ctx, req) - if err != nil { - logger.Error(). - Err(err). - Msg("droplet create failed") - return nil, err - } - - logger.Info(). - Str("name", droplet.Name). - Msg("droplet create success") - - server := &autoscaler.Server{ - Provider: autoscaler.ProviderDigitalOcean, - UID: strconv.Itoa(droplet.ID), - Name: opts.Name, - Size: req.Size, - Region: req.Region, - Image: req.Image.Slug, - Capacity: opts.Capacity, - Secret: opts.Secret, - } - - for { - logger.Debug(). - Str("name", droplet.Name). - Msg("droplet network check") - - droplet, _, err = client.Droplets.Get(ctx, droplet.ID) - if err != nil { - logger.Error(). - Err(err). - Msg("droplet details unavailable") - return nil, err - } - - for _, network := range droplet.Networks.V4 { - if network.Type == "public" { - server.Address = network.IPAddress - } - } - - if server.Address != "" { - break - } - - logger.Debug(). - Str("name", droplet.Name). - Msg("droplet network not available") - - time.Sleep(5 * time.Second) - } - - logger.Debug(). - Str("name", droplet.Name). - Str("ip", server.Address). - Msg("droplet network ready") - - server.Secret = uniuri.New() - server.Created = time.Now().Unix() - server.Updated = time.Now().Unix() - - script, err := scripts.GenerateInstall(p.config, server) - if err != nil { - return server, err - } - - // ping the server in a loop until we can successfully - // authenticate. - for i := 0; i < 20; i++ { - logger.Debug(). - Str("name", droplet.Name). - Str("ip", server.Address). - Str("port", "22"). - Str("user", "root"). - Msg("ping server") - _, err = sshutil.Execute(server.Address, "22", "root", "whoami", signer) - if err == nil { - break - } else { - time.Sleep(5 * time.Second) - } - } - - logger.Debug(). - Str("name", droplet.Name). - Str("ip", server.Address). - Msg("install agent") - - out, err := sshutil.Execute(server.Address, "22", "root", script, signer) - server.Logs = string(out) - if err != nil { - logger.Error(). - Err(err). - Str("name", droplet.Name). - Str("ip", server.Address). - Msg("install failed") - return server, err - } - - logger.Debug(). - Str("name", droplet.Name). - Str("ip", server.Address). - Msg("install complete") - - return server, nil -} - -// Destroy destroyes the DigitalOcean instance. -func (p *Provider) Destroy(ctx context.Context, server *autoscaler.Server) error { - logger := log.Ctx(ctx).With(). - Str("region", server.Region). - Str("image", server.Image). - Str("size", server.Size). - Str("name", server.Name). - Logger() - - script, err := scripts.GenerateTeardown(p.config) - if err != nil { - return err - } - - signer, err := sshutil.ParsePrivateKey(p.config.DigitalOcean.SSHKey) - if err != nil { - return err - } - - logger.Debug(). - Msg("teardown droplet") - - _, err = sshutil.Execute(server.Address, "22", "root", script, signer) - if err != nil { - logger.Error(). - Err(err). - Msg("teardown failed") - return err - } - - logger.Debug(). - Msg("deleting droplet") - - client := newClient(ctx, p.config.DigitalOcean.Token) - id, err := strconv.Atoi(server.UID) - if err != nil { - return err - } - - _, err = client.Droplets.Delete(ctx, id) - if err != nil { - logger.Error(). - Err(err). - Msg("deleting droplet failed") - return err - } - - logger.Debug(). - Msg("droplet deleted") - - return nil -} - -// helper function returns a new digitalocean client. -func newClient(ctx context.Context, token string) *godo.Client { - return godo.NewClient( - oauth2.NewClient(ctx, oauth2.StaticTokenSource( - &oauth2.Token{ - AccessToken: token, - }, - )), - ) -} diff --git a/drivers/digitalocean/provider.go b/drivers/digitalocean/provider.go new file mode 100644 index 00000000..98d2a6b5 --- /dev/null +++ b/drivers/digitalocean/provider.go @@ -0,0 +1,64 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package digitalocean + +import ( + "context" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/autoscaler/drivers/internal/base" + "github.com/drone/autoscaler/drivers/internal/scripts" + "github.com/drone/autoscaler/drivers/internal/sshutil" + + "github.com/digitalocean/godo" + "golang.org/x/crypto/ssh" + "golang.org/x/oauth2" +) + +// Provider defines the DigitalOcean provider. +type Provider struct { + autoscaler.Provider + + signer ssh.Signer + config config.Config +} + +// FromConfig loads the provider from the configuration. +func FromConfig(config config.Config) (autoscaler.Provider, error) { + signer, err := sshutil.ParsePrivateKey(config.DigitalOcean.SSHKey) + if err != nil { + return nil, err + } + return &Provider{ + Provider: base.Provider("root", "22", signer), + signer: signer, + config: config, + }, nil +} + +func (p *Provider) setupScriptOpts(instance *autoscaler.Instance) scripts.SetupOpts { + opts := scripts.SetupOpts{} + opts.Server.Host = p.config.Agent.Host + opts.Server.Secret = p.config.Agent.Token + opts.Agent.Image = p.config.Agent.Image + opts.Agent.Capacity = p.config.Agent.Concurrency + opts.Instance.Addr = instance.Address + opts.Instance.Name = instance.Name + opts.Cadvisor.Disable = false + opts.Cadvisor.Secret = instance.Secret + return opts +} + +// helper function returns a new digitalocean client. +func newClient(ctx context.Context, token string) *godo.Client { + return godo.NewClient( + oauth2.NewClient(ctx, oauth2.StaticTokenSource( + &oauth2.Token{ + AccessToken: token, + }, + )), + ) +} diff --git a/drivers/digitalocean/digitalocean_test.go b/drivers/digitalocean/provider_test.go similarity index 100% rename from drivers/digitalocean/digitalocean_test.go rename to drivers/digitalocean/provider_test.go diff --git a/drivers/hetznercloud/create.go b/drivers/hetznercloud/create.go new file mode 100644 index 00000000..fa7c3f74 --- /dev/null +++ b/drivers/hetznercloud/create.go @@ -0,0 +1,140 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package hetznercloud + +import ( + "context" + "strconv" + "time" + + "github.com/dchest/uniuri" + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/drivers/internal/scripts" + + "github.com/hetznercloud/hcloud-go/hcloud" + "github.com/rs/zerolog/log" +) + +// Create creates the HetznerCloud instance. +func (p *Provider) Create(ctx context.Context, opts autoscaler.InstanceCreateOpts) (*autoscaler.Instance, error) { + req := hcloud.ServerCreateOpts{ + Name: opts.Name, + ServerType: &hcloud.ServerType{ + Name: p.config.HetznerCloud.ServerType, + }, + Image: &hcloud.Image{ + Name: p.config.HetznerCloud.Image, + }, + Datacenter: &hcloud.Datacenter{ + Name: p.config.HetznerCloud.Datacenter, + }, + SSHKeys: []*hcloud.SSHKey{ + &hcloud.SSHKey{ + ID: p.config.HetznerCloud.SSHKeyID, + }, + }, + } + if req.ServerType.Name == "" { + req.ServerType.Name = "cx11" + } + if req.Image.Name == "" { + req.Image.Name = "ubuntu-16.04" + } + if req.Datacenter.Name == "" { + req.Datacenter.Name = "nbg1-dc3" + } + + logger := log.Ctx(ctx).With(). + Str("datacenter", req.Datacenter.Name). + Str("image", req.Image.Name). + Str("serverType", req.ServerType.Name). + Str("name", req.Name). + Logger() + + logger.Debug(). + Msg("instance create") + + client := newClient(ctx, p.config.HetznerCloud.Token) + resp, _, err := client.Server.Create(ctx, req) + if err != nil { + logger.Error(). + Err(err). + Msg("instance create failed") + return nil, err + } + + instance := &autoscaler.Instance{ + Provider: autoscaler.ProviderHetznerCloud, + ID: strconv.Itoa(resp.Server.ID), + Name: resp.Server.Name, + Address: resp.Server.PublicNet.IPv4.IP.String(), + Size: req.ServerType.Name, + Region: req.Datacenter.Name, + Image: req.Image.Name, + Secret: uniuri.New(), + } + + logger.Info(). + Str("name", instance.Name). + Msg("instance create success") + + // ping the server in a loop until we can successfully + // authenticate. + interval := time.Duration(0) +pinger: + for { + select { + case <-ctx.Done(): + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Str("port", "22"). + Str("user", "root"). + Msg("ping deadline exceeded") + + return instance, ctx.Err() + case <-time.After(interval): + interval = time.Minute + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Str("port", "22"). + Str("user", "root"). + Msg("ping server") + + err = p.Provider.Ping(ctx, instance) + if err == nil { + break pinger + } + } + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install agent") + + script, err := scripts.GenerateSetup(p.setupScriptOpts(instance)) + if err != nil { + return instance, err + } + + logs, err := p.Provider.Execute(ctx, instance, script) + if err != nil { + logger.Error(). + Err(err). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install failed") + return instance, &autoscaler.InstanceError{Err: err, Logs: logs} + } + + logger.Debug(). + Str("name", instance.Name). + Str("ip", instance.Address). + Msg("install complete") + + return instance, nil +} diff --git a/drivers/hetznercloud/create_test.go b/drivers/hetznercloud/create_test.go new file mode 100644 index 00000000..a1b0400a --- /dev/null +++ b/drivers/hetznercloud/create_test.go @@ -0,0 +1,330 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package hetznercloud + +import ( + "bytes" + "context" + "errors" + "testing" + "time" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/autoscaler/mocks" + + "github.com/golang/mock/gomock" + "github.com/h2non/gock" + "golang.org/x/crypto/ssh" +) + +func TestCreate(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.hetzner.cloud"). + Post("/v1/servers"). + Reply(200). + BodyString(respInstanceCreate) + + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Ping(gomock.Any(), gomock.Any()).Return(nil) + mockProvider.EXPECT().Execute(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + instance, err := p.Create(context.TODO(), autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err != nil { + t.Error(err) + } + + t.Run("Attributes", testInstance(instance)) +} + +func TestCreate_CreateError(t *testing.T) { + defer gock.Off() + + gock.New("https://api.hetzner.cloud"). + Post("/v1/servers"). + Reply(500) + + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + p := Provider{ + Provider: nil, + config: mockConfig, + signer: mockSigner, + } + + _, err := p.Create(context.TODO(), autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err == nil { + t.Errorf("Expect error returned from hetzner cloud") + } +} + +func TestCreate_PingTimeout(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.hetzner.cloud"). + Post("/v1/servers"). + Reply(200). + BodyString(respInstanceCreate) + + mockError := errors.New("oh no") + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Ping(ctx, gomock.Any()).Return(mockError) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + instance, err := p.Create(ctx, autoscaler.InstanceCreateOpts{Name: "agent1"}) + if err == nil { + t.Errorf("Expected context deadline exceeded, got nil") + } else if err.Error() != "context deadline exceeded" { + t.Errorf("Expected context deadline exceeded, got %s", err) + } + + t.Run("Attributes", testInstance(instance)) + t.Run("Address", testInstanceAddress(instance)) +} + +func TestCreate_ExecError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.hetzner.cloud"). + Post("/v1/servers"). + Reply(200). + BodyString(respInstanceCreate) + + mockContext := context.Background() + mockLogs := []byte("-bash: curl: command not found") + mockError := errors.New("uh oh") + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Ping(mockContext, gomock.Any()).Return(nil) + mockProvider.EXPECT().Execute(mockContext, gomock.Any(), gomock.Any()).Return(mockLogs, mockError) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + instance, err := p.Create(context.TODO(), autoscaler.InstanceCreateOpts{Name: "agent1"}) + if lerr, ok := err.(*autoscaler.InstanceError); !ok { + t.Errorf("Want InstanceError") + } else if err == nil { + t.Errorf("Want InstanceError got nil") + } else if lerr.Err != mockError { + t.Errorf("Want InstanceError to wrap the ssh error") + } else if !bytes.Equal(lerr.Logs, mockLogs) { + t.Errorf("Want InstanceError to include the logs") + } + + t.Run("Attributes", testInstance(instance)) + t.Run("Address", testInstanceAddress(instance)) +} + +func testInstance(instance *autoscaler.Instance) func(t *testing.T) { + return func(t *testing.T) { + if instance == nil { + t.Errorf("Expect non-nil instance even if error") + } + if got, want := instance.ID, "544037"; got != want { + t.Errorf("Want instance ID %v, got %v", want, got) + } + if got, want := instance.Image, "ubuntu-16.04"; got != want { + t.Errorf("Want instance Image %v, got %v", want, got) + } + if got, want := instance.Name, "test"; got != want { + t.Errorf("Want instance Name %v, got %v", want, got) + } + if got, want := instance.Region, "nbg1-dc3"; got != want { + t.Errorf("Want instance Region %v, got %v", want, got) + } + if got, want := instance.Provider, autoscaler.ProviderHetznerCloud; got != want { + t.Errorf("Want instance Provider %v, got %v", want, got) + } + if instance.Secret == "" { + t.Errorf("Want instance secret populated, got empty") + } + } +} + +func testInstanceAddress(instance *autoscaler.Instance) func(t *testing.T) { + return func(t *testing.T) { + if instance == nil { + t.Errorf("Expect non-nil instance even if error") + } + if got, want := instance.Address, "195.201.93.137"; got != want { + t.Errorf("Want instance Address %v, got %v", want, got) + } + } +} + +// sample response for POST /v1/servers +const respInstanceCreate = ` +{ + "server": { + "id": 544037, + "name": "test", + "status": "initializing", + "created": "2018-03-02T08:44:07+00:00", + "public_net": { + "ipv4": { + "ip": "195.201.93.137", + "blocked": false, + "dns_ptr": "static.137.93.201.195.clients.your-server.de" + }, + "ipv6": { + "ip": "2a01:4f8:1c0c:6996::/64", + "blocked": false, + "dns_ptr": [] + }, + "floating_ips": [] + }, + "server_type": { + "id": 1, + "name": "cx11", + "description": "CX11", + "cores": 1, + "memory": 2.0, + "disk": 20, + "prices": [ + { + "location": "fsn1", + "price_hourly": { + "net": "0.0040000000", + "gross": "0.0047600000000000" + }, + "price_monthly": { + "net": "2.4900000000", + "gross": "2.9631000000000000" + } + }, + { + "location": "nbg1", + "price_hourly": { + "net": "0.0040000000", + "gross": "0.0047600000000000" + }, + "price_monthly": { + "net": "2.4900000000", + "gross": "2.9631000000000000" + } + } + ], + "storage_type": "local" + }, + "datacenter": { + "id": 2, + "name": "nbg1-dc3", + "description": "Nuremberg 1 DC 3", + "location": { + "id": 2, + "name": "nbg1", + "description": "Nuremberg DC Park 1", + "country": "DE", + "city": "Nuremberg", + "latitude": 49.452102, + "longitude": 11.076665 + }, + "server_types": { + "supported": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ], + "available": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ] + } + }, + "image": { + "id": 1, + "type": "system", + "status": "available", + "name": "ubuntu-16.04", + "description": "Ubuntu 16.04", + "image_size": null, + "disk_size": 5, + "created": "2018-01-15T11:34:45+00:00", + "created_from": null, + "bound_to": null, + "os_flavor": "ubuntu", + "os_version": "16.04", + "rapid_deploy": true + }, + "iso": null, + "rescue_enabled": false, + "locked": false, + "backup_window": null, + "outgoing_traffic": 0, + "ingoing_traffic": 0, + "included_traffic": 21990232555520 + }, + "action": { + "id": 279192, + "command": "create_server", + "status": "running", + "progress": 0, + "started": "2018-03-02T08:44:07+00:00", + "finished": null, + "resources": [ + { + "id": 544037, + "type": "server" + } + ], + "error": null + }, + "root_password": null +} +` diff --git a/drivers/hetznercloud/destroy.go b/drivers/hetznercloud/destroy.go new file mode 100644 index 00000000..b95ff1ab --- /dev/null +++ b/drivers/hetznercloud/destroy.go @@ -0,0 +1,67 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package hetznercloud + +import ( + "context" + "strconv" + + "github.com/drone/autoscaler" + + "github.com/hetznercloud/hcloud-go/hcloud" + "github.com/rs/zerolog/log" +) + +// Destroy destroyes the HetznerCloud instance. +func (p *Provider) Destroy(ctx context.Context, instance *autoscaler.Instance) error { + logger := log.Ctx(ctx).With(). + Str("region", instance.Region). + Str("image", instance.Image). + Str("size", instance.Size). + Str("name", instance.Name). + Logger() + + client := newClient(ctx, p.config.HetznerCloud.Token) + id, err := strconv.Atoi(instance.ID) + if err != nil { + return err + } + + logger.Debug(). + Msg("teardown instance") + + _, err = p.Provider.Execute(ctx, instance, teardownScript) + if err != nil { + // if we cannot gracefully shutdown the agent we should + // still continue and destroy the instance. I think. + logger.Error(). + Err(err). + Msg("teardown failed") + + // TODO(bradrydzewski) we should snapshot the error logs + } + + logger.Debug(). + Msg("deleting instance") + + _, err = client.Server.Delete(ctx, &hcloud.Server{ID: id}) + if err != nil { + logger.Error(). + Err(err). + Msg("deleting instance failed") + return err + } + + logger.Debug(). + Msg("instance deleted") + + return nil +} + +var teardownScript = ` +set -x; +docker stop -t 3600 agent +docker ps -a +` diff --git a/drivers/hetznercloud/destroy_test.go b/drivers/hetznercloud/destroy_test.go new file mode 100644 index 00000000..379a85eb --- /dev/null +++ b/drivers/hetznercloud/destroy_test.go @@ -0,0 +1,159 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package hetznercloud + +import ( + "context" + "errors" + "strconv" + "testing" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/autoscaler/mocks" + + "github.com/golang/mock/gomock" + "github.com/h2non/gock" + "golang.org/x/crypto/ssh" +) + +func TestDestroy(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.hetzner.cloud"). + Delete("/v1/servers/3164494"). + Reply(200) + + mockContext := context.TODO() + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + mockInstance := &autoscaler.Instance{ + ID: "3164494", + } + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Execute(mockContext, mockInstance, gomock.Any()).Return(nil, nil) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + err := p.Destroy(mockContext, mockInstance) + if err != nil { + t.Error(err) + } +} + +func TestDestroyShutdownError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.hetzner.cloud"). + Delete("/v1/servers/3164494"). + Reply(200) + + mockError := errors.New("oh no") + mockContext := context.TODO() + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + mockInstance := &autoscaler.Instance{ + ID: "3164494", + } + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Execute(mockContext, mockInstance, gomock.Any()).Return(nil, mockError) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + err := p.Destroy(mockContext, mockInstance) + if err != nil { + t.Error(err) + } +} + +func TestDestroyDeleteError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + defer gock.Off() + + gock.New("https://api.hetzner.cloud"). + Delete("/v1/servers/3164494"). + Reply(500) + + mockContext := context.TODO() + mockSigner, _ := ssh.ParsePrivateKey(testkey) + mockConfig := config.Config{} + mockInstance := &autoscaler.Instance{ + ID: "3164494", + } + + // base provider to mock SSH calls. + mockProvider := mocks.NewMockProvider(controller) + mockProvider.EXPECT().Execute(mockContext, mockInstance, gomock.Any()).Return(nil, nil) + + p := Provider{ + Provider: mockProvider, + config: mockConfig, + signer: mockSigner, + } + + err := p.Destroy(mockContext, mockInstance) + if err == nil { + t.Errorf("Expect error returned from hetzner cloud") + } +} + +func TestDestroyInvalidInput(t *testing.T) { + i := &autoscaler.Instance{} + p := Provider{} + err := p.Destroy(context.TODO(), i) + if _, ok := err.(*strconv.NumError); !ok { + t.Errorf("Expected invalid or missing ID error") + } +} + +var testkey = []byte(` +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEAwz+uIrhrf/C+Ku0EaofJQvPrmkQpeV2Bx3zVWjJ7GGn1k8GQ +bGXhNEsvzV3XAoFEkSD3uc6W5fU0/HxdbLtxFxUy2dg8tO/AmTrVsHAtZBMjS6ha +2ada/Xg/iH2ZvWj2t5XCAcOpotl5SBafZYytfWal2IEQ/qBEm8olZqyo3GGPxCBn +vUspbJt/i4Nzp/0mQmUrwv5EvaJa8G0ILt/AmYrKsFY2VCTt8v4XRJaPYejSyTDt +DddH6HWxIzc5fn0WMD+nCLb+a7Q33RPfZ7Y2CmuG088INEwYNsURYVLGzHwNkT7a +lHmDVH2+0YgU08m9bnEFTkcL1aH/HVIN/tXe7wIDAQABAoIBAF1FL19gr+HHVGDX +JrPpN8inExZ3l0Rl2dg9FwJmeQ05mNnDrsVJieJcRHKbcFm+/M1DbXOyb71cfLpc +gpitliGLu+X6+U0J9vx78Za+j8Btr/+1ZejxnHLXHaqLLYUg/jLG9I25NXEY6Gn6 +fJybLkloXrNlPIQWdY/iail5M5VKo/CtOuyoSNJzN2HShe+uU4CR6js4URK7QiAA +y6dUW4VtkYlZCOATqHIMUAIx5fi/734v5b8ABUHOFNpaLBbUgKyvPS0H4MatjAJf +n4MiEj0A+Fyvv1UiXQJV5uQ5/Z3mv4Zf9dxZ6qHbGEzdGZWB19AClYTwKYfu/odu +IK5ubjECgYEA6uzRiDGxnB/Xfbb19O4WDgGut/qwP9qm+2/z08HFRv8n1VGSXsWY +AYl0VNAYbovOCGHGzhubYWbw0RenYaL/9YPiWa+ealMbDf6YsmU+0pvXuzyFtYI9 +RHIlP81ViiDXcLzu/H7BVvEv4DfHSD9jFkWMlc83TCjQX/kEuSnulUcCgYEA1MOs +bB5V/Sw4dVj/a46zdo9ZK3JkTrNnqdI8nYqlqqSO4L11i2tJWhoh5ueDJ+NLiEIu +Wujdba7I3LzyetGITTREsLShPmM0EIuX0jJpeeTO3ylaUJppiummnsGmqMfVqdik +UYrwlD0ekm5vko7tUZqmEIgVXNA2kGDyNpR93RkCgYAUP997vtTRYUlA09F1kEQk +Zu65ewlQJ7e2+ppoyU4I5Zt4XrSgKKYGk+OMH/fLJ4/V1x+8ylJlXesqCsDpwJQR +hJGxK1sbTRiK50QgNGvq2XYJ9JiN4bEIQlKFolxaMKSBWje7We2uYdG/oO8zggs3 +cz0/+IGKtgXoD93hXATtpwKBgFyPb9Btdh05AqrSd/Pz1dErVbCYCFlQpTV099fV +vHK7Okk9QwjPOM8Q9VS9vQo6UN7LY9061zHjSxD0xkx2IWTs60Ewo8E/aSQVhov0 +UHyt9O2S0O6l7mp3cXw5ZOaiYSqNzBaJalYjLMypbLKGqWnJ7JreiOSi1EoFUvo5 +qXPpAoGAVZCcIes93+9MHbcbhVyHv3X0XulrPQMRcve0M4MarZ7UWAIPrcIKhQC9 +2M+S1L0BJaHZxTQyxCb2XNRKqzQnmuhrT/A+tH8SUpl2ZLlF1KiNn4UsznBrvsaJ +OoKlArIydogC0Ugu2LoKMw8oIkpf2ANTii3dYJN6ulBKE6EYaRk= +-----END RSA PRIVATE KEY----- +`) diff --git a/drivers/hetznercloud/provider.go b/drivers/hetznercloud/provider.go new file mode 100644 index 00000000..109b567f --- /dev/null +++ b/drivers/hetznercloud/provider.go @@ -0,0 +1,61 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package hetznercloud + +import ( + "context" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/autoscaler/drivers/internal/base" + "github.com/drone/autoscaler/drivers/internal/scripts" + "github.com/drone/autoscaler/drivers/internal/sshutil" + + "github.com/hetznercloud/hcloud-go/hcloud" + "golang.org/x/crypto/ssh" +) + +// Provider defines the HetznerCloud provider. +type Provider struct { + autoscaler.Provider + + signer ssh.Signer + config config.Config +} + +// FromConfig loads the provider from the configuration. +func FromConfig(config config.Config) (autoscaler.Provider, error) { + signer, err := sshutil.ParsePrivateKey(config.HetznerCloud.SSHKey) + if err != nil { + return nil, err + } + return &Provider{ + Provider: base.Provider("root", "22", signer), + signer: signer, + config: config, + }, nil +} + +func (p *Provider) setupScriptOpts(instance *autoscaler.Instance) scripts.SetupOpts { + opts := scripts.SetupOpts{} + opts.Server.Host = p.config.Agent.Host + opts.Server.Secret = p.config.Agent.Token + opts.Agent.Image = p.config.Agent.Image + opts.Agent.Capacity = p.config.Agent.Concurrency + opts.Instance.Addr = instance.Address + opts.Instance.Name = instance.Name + opts.Cadvisor.Disable = false + opts.Cadvisor.Secret = instance.Secret + return opts +} + +// helper function returns a new HetznerCloud client. +func newClient(ctx context.Context, token string) *hcloud.Client { + return hcloud.NewClient( + hcloud.WithToken( + token, + ), + ) +} diff --git a/drivers/hetznercloud/provider_test.go b/drivers/hetznercloud/provider_test.go new file mode 100644 index 00000000..c4b4936b --- /dev/null +++ b/drivers/hetznercloud/provider_test.go @@ -0,0 +1,5 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package hetznercloud diff --git a/drivers/internal/base/provider.go b/drivers/internal/base/provider.go new file mode 100644 index 00000000..f748718f --- /dev/null +++ b/drivers/internal/base/provider.go @@ -0,0 +1,80 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package base + +import ( + "context" + "net" + "time" + + "github.com/drone/autoscaler" + "golang.org/x/crypto/ssh" +) + +// Provider returns a base Provider. +func Provider(username, port string, signer ssh.Signer) autoscaler.Provider { + return &provider{ + username: username, + port: port, + timeout: time.Minute, + signer: signer, + } +} + +type provider struct { + username string + port string + timeout time.Duration + signer ssh.Signer +} + +func (p *provider) Create(ctx context.Context, opts autoscaler.InstanceCreateOpts) (*autoscaler.Instance, error) { + panic("not implemented") +} + +func (p *provider) Destroy(ctx context.Context, instance *autoscaler.Instance) error { + panic("not implemented") +} + +func (p *provider) Execute(ctx context.Context, instance *autoscaler.Instance, command string) ([]byte, error) { + return p.execute(ctx, instance.Address, command) +} + +func (p *provider) Ping(ctx context.Context, instance *autoscaler.Instance) error { + // Hosting providers may block ping requests and some ping libraries + // require special linux capabilities. Using SSH to verify connectivity. + out, err := p.execute(ctx, instance.Address, "whoami") + if err != nil { + err = &autoscaler.InstanceError{ + Err: err, + Logs: out, + } + } + return err +} + +func (p *provider) execute(ctx context.Context, address, command string) ([]byte, error) { + config := &ssh.ClientConfig{ + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + Timeout: p.timeout, + User: p.username, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(p.signer), + }, + } + + addr := net.JoinHostPort(address, p.port) + conn, err := ssh.Dial("tcp", addr, config) + if err != nil { + return nil, err + } + defer conn.Close() + + sess, err := conn.NewSession() + if err != nil { + return nil, err + } + return sess.CombinedOutput(command) +} diff --git a/drivers/internal/certs/cert.go b/drivers/internal/certs/cert.go new file mode 100644 index 00000000..8b8d36bd --- /dev/null +++ b/drivers/internal/certs/cert.go @@ -0,0 +1,150 @@ +// Copyright Docker.IO, Inc. All rights reserved. +// https://github.com/docker/machine + +package certs + +import ( + "bytes" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "time" +) + +const ( + // default key size. + size = 2048 + + // default organization name for certificates. + organization = "drone.autoscaler.generated" +) + +// Certificate stores a certificate and private key. +type Certificate struct { + Cert []byte + Key []byte +} + +// GenerateCert generates a certificate for the host address. +func GenerateCert(host string, ca *Certificate) (*Certificate, error) { + template, err := newCertificate(organization) + if err != nil { + return nil, err + } + + if ip := net.ParseIP(host); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, host) + } + + tlsCert, err := tls.X509KeyPair(ca.Cert, ca.Key) + if err != nil { + return nil, err + } + + priv, err := rsa.GenerateKey(rand.Reader, size) + if err != nil { + return nil, err + } + + x509Cert, err := x509.ParseCertificate(tlsCert.Certificate[0]) + if err != nil { + return nil, err + } + + derBytes, err := x509.CreateCertificate( + rand.Reader, template, x509Cert, &priv.PublicKey, tlsCert.PrivateKey) + if err != nil { + return nil, err + } + + certOut := new(bytes.Buffer) + pem.Encode(certOut, &pem.Block{ + Type: "CERTIFICATE", + Bytes: derBytes, + }) + + keyOut := new(bytes.Buffer) + pem.Encode(keyOut, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }) + + return &Certificate{ + Cert: certOut.Bytes(), + Key: keyOut.Bytes(), + }, nil +} + +// GenerateCA generates a CA certificate. +func GenerateCA() (*Certificate, error) { + template, err := newCertificate(organization) + if err != nil { + return nil, err + } + + template.IsCA = true + template.KeyUsage |= x509.KeyUsageCertSign + template.KeyUsage |= x509.KeyUsageKeyEncipherment + template.KeyUsage |= x509.KeyUsageKeyAgreement + + priv, err := rsa.GenerateKey(rand.Reader, size) + if err != nil { + return nil, err + } + + derBytes, err := x509.CreateCertificate( + rand.Reader, template, template, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + certOut := new(bytes.Buffer) + pem.Encode(certOut, &pem.Block{ + Type: "CERTIFICATE", + Bytes: derBytes, + }) + + keyOut := new(bytes.Buffer) + pem.Encode(keyOut, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }) + + return &Certificate{ + Cert: certOut.Bytes(), + Key: keyOut.Bytes(), + }, nil +} + +func newCertificate(org string) (*x509.Certificate, error) { + now := time.Now() + // need to set notBefore slightly in the past to account for time + // skew in the VMs otherwise the certs sometimes are not yet valid + notBefore := time.Date(now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute()-5, 0, 0, time.Local) + notAfter := notBefore.Add(time.Hour * 24 * 1080) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, err + } + + return &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{org}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageKeyAgreement, + BasicConstraintsValid: true, + }, nil +} diff --git a/drivers/internal/certs/cert_test.go b/drivers/internal/certs/cert_test.go new file mode 100644 index 00000000..a7490412 --- /dev/null +++ b/drivers/internal/certs/cert_test.go @@ -0,0 +1,21 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package certs + +import ( + "testing" +) + +func TestGenerate(t *testing.T) { + ca, err := GenerateCA() + if err != nil { + t.Error(err) + } + + _, err = GenerateCert("159.65.43.12", ca) + if err != nil { + t.Error(err) + } +} diff --git a/drivers/internal/scripts/scripts.go b/drivers/internal/scripts/scripts.go deleted file mode 100644 index c5aea398..00000000 --- a/drivers/internal/scripts/scripts.go +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package scripts - -import ( - "bytes" - "text/template" - - "github.com/drone/autoscaler" - "github.com/drone/autoscaler/config" -) - -// GenerateInstall generates an installation script. -func GenerateInstall(config config.Config, server *autoscaler.Server) (string, error) { - buf := new(bytes.Buffer) - err := installT.Execute(buf, map[string]interface{}{ - "Server": server, - "Config": config, - }) - return buf.String(), err -} - -// GenerateTeardown generates a teardown script. -func GenerateTeardown(config config.Config) (string, error) { - buf := new(bytes.Buffer) - err := teardownT.Execute(buf, map[string]interface{}{ - "Config": config, - }) - return buf.String(), err -} - -var teardownT = template.Must(template.New("_").Funcs(funcs).Parse(` -set -x; - -sudo docker ps -sudo docker stop -t 3600 agent -sudo docker ps -a -`)) - -var installT = template.Must(template.New("_").Funcs(funcs).Parse(` -set -x; -set -e; - -if ! [ -x "$(command -v docker)" ]; then - curl -fsSL get.docker.com -o get-docker.sh - sh get-docker.sh - sudo usermod -aG docker $(whoami) -fi - -echo -n 'admin:{SHA}{{sha .Server.Secret}}' > $HOME/.htpasswd; - -sudo docker run \ ---volume=/:/rootfs:ro \ ---volume=/var/run:/var/run:rw \ ---volume=/sys:/sys:ro \ ---volume=/var/lib/docker/:/var/lib/docker:ro \ ---volume=/dev/disk/:/dev/disk:ro \ ---volume=$HOME/.htpasswd:/root/.htpasswd \ ---publish=8080:8080 \ ---detach=true \ ---name=cadvisor \ -google/cadvisor:latest \ ---http_auth_realm localhost \ ---http_auth_file /root/.htpasswd; - -sudo docker run \ ---detach=true \ ---restart=always \ ---volume /var/run/docker.sock:/var/run/docker.sock \ --e DRONE_SECRET={{.Config.Agent.Token}} \ --e DRONE_SERVER={{.Config.Agent.Host}} \ --e DRONE_MAX_PROCS={{.Server.Capacity}} \ --e DRONE_HOSTNAME={{.Server.Name}} \ ---name=agent \ -{{.Config.Agent.Image}}; - -sudo docker ps; -`)) diff --git a/drivers/internal/scripts/setup.go b/drivers/internal/scripts/setup.go new file mode 100644 index 00000000..38f0d743 --- /dev/null +++ b/drivers/internal/scripts/setup.go @@ -0,0 +1,78 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package scripts + +import ( + "bytes" + "text/template" +) + +// SetupOpts provides options for generating the setup +// shell script. +type SetupOpts struct { + Instance struct { + Name string + Addr string + } + Server struct { + Host string + Secret string + } + Agent struct { + Image string + Capacity int + } + Cadvisor struct { + Secret string + Disable bool + } +} + +// GenerateSetup generates the agent setup script. +func GenerateSetup(opts SetupOpts) (string, error) { + buf := new(bytes.Buffer) + err := setupT.Execute(buf, &opts) + return buf.String(), err +} + +var setupT = template.Must(template.New("_").Funcs(funcs).Parse(` +set -x; +set -e; + +if ! [ -x "$(command -v docker)" ]; then + curl -fsSL get.docker.com -o get-docker.sh + sh get-docker.sh + sudo usermod -aG docker $(whoami) +fi + +echo -n 'admin:{SHA}{{sha .Cadvisor.Secret}}' > $HOME/.htpasswd; + +sudo docker run \ +--volume=/:/rootfs:ro \ +--volume=/var/run:/var/run:rw \ +--volume=/sys:/sys:ro \ +--volume=/var/lib/docker/:/var/lib/docker:ro \ +--volume=/dev/disk/:/dev/disk:ro \ +--volume=$HOME/.htpasswd:/root/.htpasswd \ +--publish=8080:8080 \ +--detach=true \ +--name=cadvisor \ +google/cadvisor:latest \ +--http_auth_realm localhost \ +--http_auth_file /root/.htpasswd; + +sudo docker run \ +--detach=true \ +--restart=always \ +--volume /var/run/docker.sock:/var/run/docker.sock \ +-e DRONE_SECRET={{.Server.Secret}} \ +-e DRONE_SERVER={{.Server.Host}} \ +-e DRONE_MAX_PROCS={{.Agent.Capacity}} \ +-e DRONE_HOSTNAME={{.Instance.Name}} \ +--name=agent \ +{{.Agent.Image}}; + +sudo docker ps; +`)) diff --git a/drivers/internal/scripts/setup_test.go b/drivers/internal/scripts/setup_test.go new file mode 100644 index 00000000..aebf37f4 --- /dev/null +++ b/drivers/internal/scripts/setup_test.go @@ -0,0 +1,81 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package scripts + +import ( + "testing" + + "github.com/pmezard/go-difflib/difflib" +) + +func TestGenerateSetup(t *testing.T) { + opts := SetupOpts{} + opts.Server.Host = "localhost:9000" + opts.Server.Secret = "a8842634682b789" + opts.Agent.Image = "drone/agent:0.8" + opts.Agent.Capacity = 2 + opts.Instance.Addr = "1.2.3.4" + opts.Instance.Name = "server1" + opts.Cadvisor.Disable = false + opts.Cadvisor.Secret = "14bb43312eada8a" + + script, err := GenerateSetup(opts) + if err != nil { + t.Error(err) + return + } + if got, want := script, setupScript; got != want { + t.Errorf("Invalid script") + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(got), + B: difflib.SplitLines(want), + FromFile: "Got", + ToFile: "Want", + Context: 5, + } + text, _ := difflib.GetUnifiedDiffString(diff) + t.Log(text) + } +} + +var setupScript = ` +set -x; +set -e; + +if ! [ -x "$(command -v docker)" ]; then + curl -fsSL get.docker.com -o get-docker.sh + sh get-docker.sh + sudo usermod -aG docker $(whoami) +fi + +echo -n 'admin:{SHA}0RIWnjGvcw2wHMferV9MJVSo0Uw=' > $HOME/.htpasswd; + +sudo docker run \ +--volume=/:/rootfs:ro \ +--volume=/var/run:/var/run:rw \ +--volume=/sys:/sys:ro \ +--volume=/var/lib/docker/:/var/lib/docker:ro \ +--volume=/dev/disk/:/dev/disk:ro \ +--volume=$HOME/.htpasswd:/root/.htpasswd \ +--publish=8080:8080 \ +--detach=true \ +--name=cadvisor \ +google/cadvisor:latest \ +--http_auth_realm localhost \ +--http_auth_file /root/.htpasswd; + +sudo docker run \ +--detach=true \ +--restart=always \ +--volume /var/run/docker.sock:/var/run/docker.sock \ +-e DRONE_SECRET=a8842634682b789 \ +-e DRONE_SERVER=localhost:9000 \ +-e DRONE_MAX_PROCS=2 \ +-e DRONE_HOSTNAME=server1 \ +--name=agent \ +drone/agent:0.8; + +sudo docker ps; +` diff --git a/drivers/internal/scripts/teardown.go b/drivers/internal/scripts/teardown.go new file mode 100644 index 00000000..763af591 --- /dev/null +++ b/drivers/internal/scripts/teardown.go @@ -0,0 +1,25 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package scripts + +import ( + "bytes" + "text/template" +) + +// GenerateTeardown generates a teardown script. +func GenerateTeardown() (string, error) { + buf := new(bytes.Buffer) + err := teardownT.Execute(buf, nil) + return buf.String(), err +} + +var teardownT = template.Must(template.New("_").Funcs(funcs).Parse(` +set -x; + +sudo docker ps +sudo docker stop -t 3600 agent +sudo docker ps -a +`)) diff --git a/drivers/internal/scripts/scripts_test.go b/drivers/internal/scripts/teardown_test.go similarity index 66% rename from drivers/internal/scripts/scripts_test.go rename to drivers/internal/scripts/teardown_test.go index 4452da1e..ecf79b05 100644 --- a/drivers/internal/scripts/scripts_test.go +++ b/drivers/internal/scripts/teardown_test.go @@ -7,46 +7,11 @@ package scripts import ( "testing" - "github.com/drone/autoscaler" - "github.com/drone/autoscaler/config" - "github.com/pmezard/go-difflib/difflib" ) -func TestGenerateInstall(t *testing.T) { - conf := config.Config{} - conf.Agent.Host = "localhost:9000" - conf.Agent.Image = "drone/agent:0.8" - conf.Agent.Token = "a8842634682b789" - - server := autoscaler.Server{} - server.Name = "server1" - server.Secret = "14bb43312eada8a" - server.Capacity = 2 - - script, err := GenerateInstall(conf, &server) - if err != nil { - t.Error(err) - return - } - if got, want := script, installScript; got != want { - t.Errorf("Invalid script") - diff := difflib.UnifiedDiff{ - A: difflib.SplitLines(got), - B: difflib.SplitLines(want), - FromFile: "Got", - ToFile: "Want", - Context: 5, - } - text, _ := difflib.GetUnifiedDiffString(diff) - t.Log(text) - } -} - func TestGenerateTeardown(t *testing.T) { - conf := config.Config{} - - script, err := GenerateTeardown(conf) + script, err := GenerateTeardown() if err != nil { t.Error(err) return diff --git a/engine.go b/engine.go new file mode 100644 index 00000000..28146b2b --- /dev/null +++ b/engine.go @@ -0,0 +1,22 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package autoscaler + +import "context" + +// An Engine is responsible for running the scaling +// alogirthm to provision and shutdown instances according +// to build volume. +type Engine interface { + // Start starts the Engine. The context can be used + // to cancel a running engine. + Start(context.Context) + // Pause pauses the Engine. + Pause() + // Paused returns true if th Engine is paused. + Paused() bool + // Resume resumes the Engine if paused. + Resume() +} diff --git a/engine/alloc.go b/engine/alloc.go new file mode 100644 index 00000000..bc654b9a --- /dev/null +++ b/engine/alloc.go @@ -0,0 +1,97 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "sync" + "time" + + "github.com/drone/autoscaler" + + "github.com/rs/zerolog/log" +) + +type allocator struct { + wg sync.WaitGroup + + servers autoscaler.ServerStore + provider autoscaler.Provider +} + +func (a *allocator) Allocate(ctx context.Context) error { + logger := log.Ctx(ctx) + + servers, err := a.servers.ListState(ctx, autoscaler.StatePending) + if err != nil { + return err + } + + for _, server := range servers { + server.State = autoscaler.StateStaging + err = a.servers.Update(ctx, server) + if err != nil { + logger.Error(). + Err(err). + Str("server", server.Name). + Str("state", "staging"). + Msg("failed to update server state") + return err + } + + a.wg.Add(1) + go func(server *autoscaler.Server) { + a.allocate(ctx, server) + a.wg.Done() + }(server) + } + return nil +} + +func (a *allocator) allocate(ctx context.Context, server *autoscaler.Server) error { + logger := log.Ctx(ctx) + defer func() { + if err := recover(); err != nil { + logger.Error(). + Err(err.(error)). + Str("server", server.Name). + Msg("unexpected panic") + } + }() + + ctx, cancel := context.WithTimeout(ctx, time.Hour) + defer cancel() + + opt := autoscaler.InstanceCreateOpts{Name: server.Name} + instance, err := a.provider.Create(ctx, opt) + if err != nil { + log.Ctx(ctx).Error(). + Err(err). + Str("server", server.Name). + Msg("failed to provision server") + + server.State = autoscaler.StateError + } else { + logger.Debug(). + Str("server", server.Name). + Msg("provisioned server") + + server.State = autoscaler.StateRunning + } + if lerr, ok := err.(*autoscaler.InstanceError); ok { + server.Error = string(lerr.Logs) + } + if instance != nil { + server.ID = instance.ID + server.Address = instance.Address + server.Image = instance.Image + server.Provider = instance.Provider + server.Region = instance.Region + server.Secret = instance.Secret + server.Size = instance.Size + server.Started = time.Now().Unix() + } + return a.servers.Update(ctx, server) +} diff --git a/engine/alloc_test.go b/engine/alloc_test.go new file mode 100644 index 00000000..d3814e65 --- /dev/null +++ b/engine/alloc_test.go @@ -0,0 +1,112 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "errors" + "testing" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/mocks" + + "github.com/golang/mock/gomock" +) + +func TestAllocate(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockInstance := &autoscaler.Instance{} + mockServers := []*autoscaler.Server{ + {State: autoscaler.StatePending}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StatePending).Return(mockServers, nil) + store.EXPECT().Update(mockctx, mockServers[0]).Return(nil) + store.EXPECT().Update(gomock.Any(), mockServers[0]).Return(nil) + + provider := mocks.NewMockProvider(controller) + provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(mockInstance, nil) + + a := allocator{servers: store, provider: provider} + err := a.Allocate(mockctx) + a.wg.Wait() + + if err != nil { + t.Error(err) + } + if got, want := mockServers[0].State, autoscaler.StateRunning; got != want { + t.Errorf("Want server state Running, got %v", got) + } +} + +func TestAllocate_ServerCreateError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockerr := errors.New("mock error") + mockServers := []*autoscaler.Server{ + {State: autoscaler.StatePending}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StatePending).Return(mockServers, nil) + store.EXPECT().Update(mockctx, mockServers[0]).Return(nil) + store.EXPECT().Update(gomock.Any(), mockServers[0]).Return(nil) + + provider := mocks.NewMockProvider(controller) + provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, mockerr) + + a := allocator{servers: store, provider: provider} + a.Allocate(mockctx) + a.wg.Wait() + + if got, want := mockServers[0].State, autoscaler.StateError; got != want { + t.Errorf("Want server state Error, got %v", got) + } +} + +func TestAllocate_ServerListError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockerr := errors.New("mock error") + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StatePending).Return(nil, mockerr) + + a := allocator{servers: store} + if got, want := a.Allocate(mockctx), mockerr; got != want { + t.Errorf("Want error getting server list") + } +} + +func TestAllocate_ServerUpdateError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockerr := errors.New("mock error") + mockServers := []*autoscaler.Server{ + {State: autoscaler.StatePending}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StatePending).Return(mockServers, nil) + store.EXPECT().Update(mockctx, mockServers[0]).Return(mockerr) + + a := allocator{servers: store} + if got, want := a.Allocate(mockctx), mockerr; got != want { + t.Errorf("Want error updating server") + } + if got, want := mockServers[0].State, autoscaler.StateStaging; got != want { + t.Errorf("Want server state Staging, got %v", got) + } +} diff --git a/engine/calc.go b/engine/calc.go new file mode 100644 index 00000000..a862b2cd --- /dev/null +++ b/engine/calc.go @@ -0,0 +1,55 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import "math" + +// helper function returns the absolute value of x. +func abs(x int) int { + if x < 0 { + x = x * -1 + } + return x +} + +// helper function returns the larger of x or y. +func max(x, y int) int { + if x > y { + return x + } + return y +} + +// helper function calculates the different between the existing +// server count and required server count to handle queue volume. +func serverDiff(pending, available, concurrency int) int { + return int( + math.Ceil( + float64(pending-available) / + float64(concurrency), + ), + ) +} + +// helper function adjusts the number of servers to provision +// to ensure it does not exceed the max server count. +func serverCeil(count, additions, ceiling int) int { + if count+additions >= ceiling { + additions = ceiling - count + } + return additions +} + +// helper function adjusts the number of servers to provision +// to ensure the minimum server count is maintained. +func serverFloor(count, deletions, floor int) int { + if deletions == 0 { + return 0 + } + if floor > count-deletions { + deletions = count - floor + } + return max(deletions, 0) +} diff --git a/engine/calc_test.go b/engine/calc_test.go new file mode 100644 index 00000000..7609d4f1 --- /dev/null +++ b/engine/calc_test.go @@ -0,0 +1,273 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import "testing" + +func TestAbs(t *testing.T) { + tests := []struct { + x, want int + }{ + {0, 0}, + {1, 1}, + {-1, 1}, + } + for _, test := range tests { + if got, want := abs(test.x), test.want; got != want { + t.Errorf("Want abs value %d, got %d", want, got) + } + } +} + +func TestMax(t *testing.T) { + tests := []struct { + x, y, want int + }{ + {0, 1, 1}, + {0, 0, 0}, + {1, 1, 1}, + {-1, 0, 0}, + {-1, 1, 1}, + } + for _, test := range tests { + if got, want := max(test.x, test.y), test.want; got != want { + t.Errorf("Want max value %d, got %d", want, got) + } + } +} + +func TestServerDiff(t *testing.T) { + tests := []struct { + pending, // count pending builds + available, // count available capacity + concurrency, // per-server concurrency + want int + }{ + // use 2 of 2 existing + { + pending: 2, + available: 2, + concurrency: 2, + want: 0, + }, + // use 1 of 2 existing + { + pending: 1, + available: 2, + concurrency: 2, + want: 0, + }, + // want 1 server + { + pending: 4, + available: 2, + concurrency: 2, + want: 1, + }, + // want 2 servers + { + pending: 4, + available: 2, + concurrency: 1, + want: 2, + }, + // want 2 servers (round-up) + { + pending: 5, + available: 2, + concurrency: 2, + want: 2, + }, + + // + // the following test cases check for instances when + // we have exceess server capacity and want to remove + // server instances. + // + + // want 0 servers removed, at capacity + { + pending: 1, + available: 1, + concurrency: 2, + want: 0, + }, + // want 0 servers removed, at capacity (server partially used) + { + pending: 1, + available: 2, + concurrency: 2, + want: 0, + }, + // want 1 server removed, pending builds, but excess capacity + { + pending: 2, + available: 4, + concurrency: 2, + want: -1, + }, + // want 2 servers removed (round down) + { + pending: 0, + available: 5, + concurrency: 2, + want: -2, + }, + // want 10 servers removed + { + pending: 4, + available: 24, + concurrency: 2, + want: -10, + }, + } + for _, test := range tests { + diff := serverDiff( + test.pending, + test.available, + test.concurrency, + ) + if got, want := diff, test.want; got != want { + t.Errorf("Got server diff %d, want %d", got, want) + } + } +} + +func TestSeverCeil(t *testing.T) { + tests := []struct { + curr, // count of servers running + diff, // count of servers to add + ceil, // max number of servers + want int + }{ + // add 0 servers + { + curr: 2, + diff: 0, + ceil: 2, + want: 0, + }, + // add 0 servers, handle 0 current count + { + curr: 0, + diff: 0, + ceil: 1, + want: 0, + }, + // add 1 server + { + curr: 2, + diff: 1, + ceil: 4, + want: 1, + }, + // add 1 server, handle 0 current count + { + curr: 0, + diff: 2, + ceil: 1, + want: 1, + }, + // add 2 servers + { + curr: 2, + diff: 2, + ceil: 4, + want: 2, + }, + // add 2 servers, adjust to ceil + { + curr: 2, + diff: 4, + ceil: 4, + want: 2, + }, + // add 4 servers, adjust to ceil + { + curr: 0, + diff: 10, + ceil: 4, + want: 4, + }, + } + for _, test := range tests { + diff := serverCeil( + test.curr, + test.diff, + test.ceil, + ) + if got, want := diff, test.want; got != want { + t.Errorf("Got server diff %d, want %d", got, want) + } + } +} + +func TestSeverFloor(t *testing.T) { + tests := []struct { + curr, // count of servers running + diff, // count of servers to remove + floor, // min number of servers + want int + }{ + // remove 0 servers + { + curr: 2, + diff: 0, + floor: 2, + want: 0, + }, + // remove 1 server + { + curr: 4, + diff: 1, + floor: 2, + want: 1, + }, + // remove 2 servers + { + curr: 4, + diff: 2, + floor: 2, + want: 2, + }, + // remove 2 servers, adjust to floor + { + curr: 4, + diff: 3, + floor: 2, + want: 2, + }, + // remove 0 servers, adjust to floor + { + curr: 2, + diff: 1, + floor: 2, + want: 0, + }, + // should not remove non-existent servers + { + curr: 0, + diff: 4, + floor: 2, + want: 0, + }, + { + curr: 1, + diff: 4, + floor: 2, + want: 0, + }, + } + for _, test := range tests { + diff := serverFloor( + test.curr, + test.diff, + test.floor, + ) + if got, want := diff, test.want; got != want { + t.Errorf("Got server diff %d, want %d", got, want) + } + } +} diff --git a/engine/collect.go b/engine/collect.go new file mode 100644 index 00000000..2cf41642 --- /dev/null +++ b/engine/collect.go @@ -0,0 +1,97 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "sync" + "time" + + "github.com/drone/autoscaler" + + "github.com/rs/zerolog/log" +) + +type collector struct { + wg sync.WaitGroup + + servers autoscaler.ServerStore + provider autoscaler.Provider +} + +func (c *collector) Collect(ctx context.Context) error { + logger := log.Ctx(ctx) + + servers, err := c.servers.ListState(ctx, autoscaler.StateShutdown) + if err != nil { + return err + } + + for _, server := range servers { + server.State = autoscaler.StateStopping + err = c.servers.Update(ctx, server) + if err != nil { + logger.Error(). + Err(err). + Str("server", server.Name). + Str("state", "stopping"). + Msg("failed to update server state") + return err + } + + c.wg.Add(1) + go func(server *autoscaler.Server) { + c.collect(ctx, server) + c.wg.Done() + }(server) + } + return nil +} + +func (c *collector) collect(ctx context.Context, server *autoscaler.Server) error { + logger := log.Ctx(ctx) + logger.Debug(). + Str("server", server.Name). + Msg("destroying server") + + defer func() { + if err := recover(); err != nil { + logger.Error(). + Err(err.(error)). + Str("server", server.Name). + Msg("unexpected panic") + } + }() + + ctx, cancel := context.WithTimeout(ctx, time.Hour) + defer cancel() + + in := &autoscaler.Instance{ + ID: server.ID, + Provider: server.Provider, + Name: server.Name, + Address: server.Address, + Region: server.Region, + Image: server.Image, + Size: server.Size, + } + err := c.provider.Destroy(ctx, in) + if err != nil { + logger.Error(). + Str("server", server.Name). + Msg("failed to destroy server") + + server.State = autoscaler.StateError + } else { + logger.Debug(). + Str("server", server.Name). + Msg("destroyed server") + + server.Stopped = time.Now().Unix() + server.State = autoscaler.StateStopped + } + + return c.servers.Update(ctx, server) +} diff --git a/engine/collect_test.go b/engine/collect_test.go new file mode 100644 index 00000000..2e2f4900 --- /dev/null +++ b/engine/collect_test.go @@ -0,0 +1,111 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "errors" + "testing" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/mocks" + + "github.com/golang/mock/gomock" +) + +func TestCollect(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockServers := []*autoscaler.Server{ + {State: autoscaler.StateShutdown}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StateShutdown).Return(mockServers, nil) + store.EXPECT().Update(mockctx, mockServers[0]).Return(nil) + store.EXPECT().Update(gomock.Any(), mockServers[0]).Return(nil) + + provider := mocks.NewMockProvider(controller) + provider.EXPECT().Destroy(gomock.Any(), gomock.Any()).Return(nil) + + c := collector{servers: store, provider: provider} + err := c.Collect(mockctx) + c.wg.Wait() + + if err != nil { + t.Error(err) + } + if got, want := mockServers[0].State, autoscaler.StateStopped; got != want { + t.Errorf("Want server state Stopped, got %v", got) + } +} + +func TestCollect_ServerDestroyError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockerr := errors.New("mock error") + mockServers := []*autoscaler.Server{ + {State: autoscaler.StateShutdown}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StateShutdown).Return(mockServers, nil) + store.EXPECT().Update(mockctx, mockServers[0]).Return(nil) + store.EXPECT().Update(gomock.Any(), mockServers[0]).Return(nil) + + provider := mocks.NewMockProvider(controller) + provider.EXPECT().Destroy(gomock.Any(), gomock.Any()).Return(mockerr) + + c := collector{servers: store, provider: provider} + c.Collect(mockctx) + c.wg.Wait() + + if got, want := mockServers[0].State, autoscaler.StateError; got != want { + t.Errorf("Want server state Error, got %v", got) + } +} + +func TestCollect_ServerListError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockerr := errors.New("mock error") + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StateShutdown).Return(nil, mockerr) + + c := collector{servers: store} + if got, want := c.Collect(mockctx), mockerr; got != want { + t.Errorf("Want error getting server list") + } +} + +func TestCollect_ServerUpdateError(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + mockctx := context.Background() + mockerr := errors.New("mock error") + mockServers := []*autoscaler.Server{ + {State: autoscaler.StateShutdown}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().ListState(mockctx, autoscaler.StateShutdown).Return(mockServers, nil) + store.EXPECT().Update(mockctx, mockServers[0]).Return(mockerr) + + c := collector{servers: store} + if got, want := c.Collect(mockctx), mockerr; got != want { + t.Errorf("Want error updating server") + } + if got, want := mockServers[0].State, autoscaler.StateStopping; got != want { + t.Errorf("Want server state Stopping, got %v", got) + } +} diff --git a/engine/engine.go b/engine/engine.go new file mode 100644 index 00000000..5ef184bc --- /dev/null +++ b/engine/engine.go @@ -0,0 +1,163 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "sync" + "time" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/drone-go/drone" + + "github.com/rs/zerolog/log" +) + +// defines the interval at which terminated instances are +// purged from the database. +const purge = time.Hour * 24 + +type engine struct { + mu sync.Mutex + + allocator *allocator + collector *collector + planner *planner + + interval time.Duration + paused bool +} + +// New returns a new autoscale Engine. +func New( + client drone.Client, + config config.Config, + servers autoscaler.ServerStore, + provider autoscaler.Provider, +) autoscaler.Engine { + return &engine{ + paused: false, + interval: config.Interval, + allocator: &allocator{ + servers: servers, + provider: provider, + }, + collector: &collector{ + servers: servers, + provider: provider, + }, + planner: &planner{ + client: client, + servers: servers, + ttu: config.Pool.MinAge, + min: config.Pool.Min, + max: config.Pool.Max, + cap: config.Agent.Concurrency, + }, + } +} + +// Pause paueses the scaler. +func (e *engine) Pause() { + e.mu.Lock() + e.paused = true + e.mu.Unlock() +} + +// Paused returns true if scaling is paused. +func (e *engine) Paused() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.paused +} + +// Resume resumes the scaler. +func (e *engine) Resume() { + e.mu.Lock() + e.paused = false + e.mu.Unlock() +} + +func (e *engine) Start(ctx context.Context) { + var wg sync.WaitGroup + wg.Add(4) + go func() { + e.allocate(ctx) + wg.Done() + }() + go func() { + e.collect(ctx) + wg.Done() + }() + go func() { + e.plan(ctx) + wg.Done() + }() + go func() { + e.purge(ctx) + wg.Done() + }() + wg.Wait() +} + +// runs the allocation process. +func (e *engine) allocate(ctx context.Context) { + const interval = time.Second * 10 + for { + select { + case <-ctx.Done(): + return + case <-time.After(interval): + e.allocator.Allocate(ctx) + } + } +} + +// runs the collection process. +func (e *engine) collect(ctx context.Context) { + const interval = time.Second * 10 + for { + select { + case <-ctx.Done(): + return + case <-time.After(interval): + e.collector.Collect(ctx) + } + } +} + +// runs the planning process. +func (e *engine) plan(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case <-time.After(e.interval): + if !e.Paused() { + e.planner.Plan(ctx) + } + } + } +} + +// runs the purge process. +func (e *engine) purge(ctx context.Context) { + const interval = time.Hour * 24 + const retain = time.Hour * 24 * -1 + + logger := log.Ctx(ctx) + for { + select { + case <-ctx.Done(): + return + case <-time.After(interval): + logger.Debug(). + Str("ttl", retain.String()). + Msg("clear stopped servers from database") + e.planner.servers.Purge(ctx, time.Now().Add(retain).Unix()) + } + } +} diff --git a/engine/option.go b/engine/option.go new file mode 100644 index 00000000..6fd6c8c1 --- /dev/null +++ b/engine/option.go @@ -0,0 +1,5 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine diff --git a/engine/option_test.go b/engine/option_test.go new file mode 100644 index 00000000..6fd6c8c1 --- /dev/null +++ b/engine/option_test.go @@ -0,0 +1,5 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine diff --git a/engine/planner.go b/engine/planner.go new file mode 100644 index 00000000..8fabf037 --- /dev/null +++ b/engine/planner.go @@ -0,0 +1,257 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "sort" + "time" + + "github.com/drone/autoscaler" + "github.com/drone/drone-go/drone" + + "github.com/dchest/uniuri" + "github.com/rs/zerolog/log" +) + +// a planner is responsible for capacity planning. It will assess +// current build volume and plan the creation or termination of +// server resources accordingly. +type planner struct { + min int // min number of servers + max int // max number of servers to allocate + cap int // capacity per-server + ttu time.Duration // minimum server age + + client drone.Client + servers autoscaler.ServerStore +} + +func (p *planner) Plan(ctx context.Context) error { + // generate a unique identifier for the current + // execution cycle for tracing and grouping logs. + cycle := uniuri.New() + + logger := log.Ctx(ctx).With().Str("id", cycle).Logger() + + pending, running, err := p.count(ctx) + if err != nil { + logger.Error().Err(err). + Msg("cannot fetch queue details") + return err + } + + capacity, servers, err := p.capacity(ctx) + if err != nil { + logger.Error().Err(err). + Msg("cannot calculate server capacity") + return err + } + + logger.Debug(). + Int("min-pool", p.min). + Int("max-pool", p.max). + Int("server-capacity", capacity). + Int("server-count", servers). + Int("pending-builds", pending). + Int("running-builds", running). + Msg("check capacity") + + defer func() { + logger.Debug(). + Msg("check capacity complete") + }() + + ctx = logger.WithContext(ctx) + + free := max(capacity-running, 0) + diff := serverDiff(pending, free, p.cap) + + // if the server differential to handle the build volume + // is positive, we can reduce server capacity. + if diff < 0 { + return p.mark(ctx, + // we should adjust the desired capacity to ensure + // we maintain the minimum required server count. + serverFloor(servers, abs(diff), p.min), + ) + } + + // if the server differential to handle the build volume + // is positive, we need to allocate more server capacity. + if diff > 0 { + return p.alloc(ctx, + // we should adjust the desired capacity to ensure + // it does not exceed the max server count. + serverCeil(servers, diff, p.max), + ) + } + + logger.Debug(). + Msg("no capacity changes required") + + return nil +} + +// helper function allocates n new server instances. +func (p *planner) alloc(ctx context.Context, n int) error { + logger := log.Ctx(ctx) + + logger.Debug(). + Msgf("allocate %d servers", n) + + for i := 0; i < n; i++ { + server := &autoscaler.Server{ + Name: "agent-" + uniuri.NewLen(8), + State: autoscaler.StatePending, + Secret: uniuri.New(), + Capacity: p.cap, + } + + err := p.servers.Create(ctx, server) + if err != nil { + logger.Error().Err(err). + Msg("cannot create server") + return err + } + } + return nil +} + +// helper funciton marks instances for termination. +func (p *planner) mark(ctx context.Context, n int) error { + logger := log.Ctx(ctx) + + logger.Debug(). + Msgf("terminate %d servers", n) + + if n == 0 { + return nil + } + + servers, err := p.servers.List(ctx) + if err != nil { + logger.Error().Err(err). + Msg("cannot fetch server list") + return err + } + sort.Sort(sort.Reverse(byCreated(servers))) + + busy, err := p.listBusy(ctx) + if err != nil { + logger.Error().Err(err). + Msg("cannot ascertain busy server list") + return err + } + + var idle []*autoscaler.Server + for _, server := range servers { + // skip busy servers + if _, ok := busy[server.Name]; ok { + logger.Debug(). + Str("server", server.Name). + Msg("server is busy") + continue + } + + // skip servers less than minage + if time.Now().Before(time.Unix(server.Created, 0).Add(p.ttu)) { + logger.Debug(). + Str("server", server.Name). + TimeDiff("age", time.Now(), time.Unix(server.Created, 0)). + Dur("min-age", p.ttu). + Msg("server min-age not reached") + continue + } + + idle = append(idle, server) + logger.Debug(). + Str("server", server.Name). + Msg("server is idle") + } + + // if there are no idle servers, there are no servers + // to retire and we can exit. + if len(idle) == 0 { + logger.Debug(). + Msg("no idle servers to shutdown") + } + + if len(idle) > n { + idle = idle[:n] + } + + for _, server := range idle { + server.State = autoscaler.StateShutdown + err := p.servers.Update(ctx, server) + if err != nil { + logger.Error(). + Err(err). + Str("server", server.Name). + Str("state", "shutdown"). + Msg("cannot update server state") + } + } + + return nil +} + +// helper function returns the number of pending and +// running builds in the remote Drone installation. +func (p *planner) count(ctx context.Context) (pending, running int, err error) { + activity, err := p.client.BuildQueue() + if err != nil { + return pending, running, err + } + for _, activity := range activity { + if activity.Status == drone.StatusPending { + pending++ + } else { + running++ + } + } + return +} + +// helper function returns our current capacity. +func (p *planner) capacity(ctx context.Context) (capacity, count int, err error) { + servers, err := p.servers.List(ctx) + if err != nil { + return capacity, count, err + } + for _, server := range servers { + // ignores stopped, stopping or errored servers. + switch server.State { + case autoscaler.StatePending, + autoscaler.StateRunning, + autoscaler.StateStaging: + count++ + capacity += server.Capacity + } + } + return +} + +// helper function returns a list of busy servers. +func (p *planner) listBusy(ctx context.Context) (map[string]struct{}, error) { + busy := map[string]struct{}{} + builds, err := p.client.BuildQueue() + if err != nil { + return busy, err + } + for _, build := range builds { + if build.Status != drone.StatusRunning { + continue + } + build, err := p.client.Build(build.Owner, build.Name, build.Number) + if err != nil { + return busy, err + } + for _, proc := range build.Procs { + busy[proc.Machine] = struct{}{} + } + } + return busy, nil +} diff --git a/engine/planner_test.go b/engine/planner_test.go new file mode 100644 index 00000000..5f99af53 --- /dev/null +++ b/engine/planner_test.go @@ -0,0 +1,403 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "context" + "testing" + "time" + + "github.com/drone/autoscaler" + "github.com/drone/autoscaler/config" + "github.com/drone/autoscaler/mocks" + "github.com/drone/drone-go/drone" + + "github.com/golang/mock/gomock" +) + +// This test verifies that if the server capacity is +// >= the pending count, and the server capacity is +// <= the pool minimum size, no actions are taken. +func TestPlan_Noop(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + servers := []*autoscaler.Server{ + {Name: "server1", Capacity: 2, State: autoscaler.StateRunning}, + {Name: "server2", Capacity: 2, State: autoscaler.StateRunning}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + + client := mocks.NewMockClient(controller) + client.EXPECT().BuildQueue().Return([]*drone.Activity{ + {Status: drone.StatusRunning}, + {Status: drone.StatusPending}, + {Status: drone.StatusPending}, + }, nil) + + p := planner{ + cap: 2, + min: 2, + max: 10, + client: client, + servers: store, + } + + err := p.Plan(context.TODO()) + if err != nil { + t.Error(err) + } +} + +// This test verifies that if the server capacity is +// < than the pending count, and the server capacity is +// >= the pool maximum, no actions are taken. +func TestPlan_MaxCapacity(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + // x4 capacity + servers := []*autoscaler.Server{ + {Name: "server1", Capacity: 1, State: autoscaler.StateRunning}, + {Name: "server2", Capacity: 1, State: autoscaler.StateRunning}, + {Name: "server3", Capacity: 1, State: autoscaler.StateRunning}, + {Name: "server4", Capacity: 1, State: autoscaler.StateRunning}, + } + + // x4 running builds + // x3 pending builds + builds := []*drone.Activity{ + {Status: drone.StatusRunning}, + {Status: drone.StatusRunning}, + {Status: drone.StatusRunning}, + {Status: drone.StatusRunning}, + {Status: drone.StatusPending}, + {Status: drone.StatusPending}, + {Status: drone.StatusPending}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + + client := mocks.NewMockClient(controller) + client.EXPECT().BuildQueue().Return(builds, nil) + + config := config.Config{} + config.Pool.Min = 2 + config.Pool.Max = 4 + config.Agent.Concurrency = 2 + + p := planner{ + cap: 2, + min: 2, + max: 4, + client: client, + servers: store, + } + + err := p.Plan(context.TODO()) + if err != nil { + t.Error(err) + } +} + +// This test verifies that if the server capacity is +// less than the pending count, and the server capacity is +// < the pool maximum, additional servers are provisioned. +func TestPlan_MoreCapacity(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + // x2 capacity + servers := []*autoscaler.Server{ + {Name: "server1", Capacity: 1, State: autoscaler.StateRunning}, + {Name: "server2", Capacity: 1, State: autoscaler.StateRunning}, + } + + // x2 running builds + // x3 pending builds + builds := []*drone.Activity{ + {Status: drone.StatusRunning}, + {Status: drone.StatusRunning}, + {Status: drone.StatusPending}, + {Status: drone.StatusPending}, + {Status: drone.StatusPending}, + {Status: drone.StatusPending}, // ignore, would exceed max pool size + {Status: drone.StatusPending}, // ignore, would exceed max pool size + {Status: drone.StatusPending}, // ignore, would exceed max pool size + {Status: drone.StatusPending}, // ignore, would exceed max pool size + {Status: drone.StatusPending}, // ignore, would exceed max pool size + {Status: drone.StatusPending}, // ignore, would exceed max pool size + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) + store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) + + client := mocks.NewMockClient(controller) + client.EXPECT().BuildQueue().Return(builds, nil) + + p := planner{ + cap: 2, + min: 2, + max: 4, + client: client, + servers: store, + } + + err := p.Plan(context.TODO()) + if err != nil { + t.Error(err) + } +} + +// This test verifies that if that no servers are +// destroyed if there is excess capacity and the +// the server count <= the min pool size. +func TestPlan_MinPool(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + // x2 capacity + servers := []*autoscaler.Server{ + {Name: "server1", Capacity: 1, State: autoscaler.StateRunning}, + {Name: "server2", Capacity: 1, State: autoscaler.StateRunning}, + } + + // x0 running builds + // x0 pending builds + builds := []*drone.Activity{} + + store := mocks.NewMockServerStore(controller) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + + client := mocks.NewMockClient(controller) + client.EXPECT().BuildQueue().Return(builds, nil) + + p := planner{ + cap: 2, + min: 2, + max: 4, + client: client, + servers: store, + } + + err := p.Plan(context.TODO()) + if err != nil { + t.Error(err) + } +} + +// This test verifies that if that no servers are +// destroyed if no idle servers exist. +func TestPlan_NoIdle(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + // x2 capacity + servers := []*autoscaler.Server{ + {Name: "server1", Capacity: 2, State: autoscaler.StateRunning}, + {Name: "server2", Capacity: 2, State: autoscaler.StateRunning}, + } + + // x2 running builds + // x0 pending builds + builds := []*drone.Activity{ + {Status: drone.StatusRunning}, + {Status: drone.StatusRunning}, + } + + store := mocks.NewMockServerStore(controller) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + + client := mocks.NewMockClient(controller) + client.EXPECT().BuildQueue().Return(builds, nil) + client.EXPECT().BuildQueue().Return(builds, nil) + client.EXPECT().Build(gomock.Any(), gomock.Any(), gomock.Any()).Return(&drone.Build{Procs: []*drone.Proc{{Machine: "server1"}}}, nil) + client.EXPECT().Build(gomock.Any(), gomock.Any(), gomock.Any()).Return(&drone.Build{Procs: []*drone.Proc{{Machine: "server2"}}}, nil) + + p := planner{ + cap: 2, + min: 1, + max: 4, + client: client, + servers: store, + } + + err := p.Plan(context.Background()) + if err != nil { + t.Error(err) + } +} + +// This test verifies that idle servers are not +// garbage collected until the min-age is reached. +func TestScale_MinAge(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + // x2 capacity + servers := []*autoscaler.Server{ + {Name: "server1", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix()}, + {Name: "server2", Capacity: 1, State: autoscaler.StateRunning, Created: time.Now().Unix()}, + } + + // x0 running builds + // x0 pending builds + builds := []*drone.Activity{} + + store := mocks.NewMockServerStore(controller) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + + client := mocks.NewMockClient(controller) + client.EXPECT().BuildQueue().Return(builds, nil) + client.EXPECT().BuildQueue().Return(builds, nil) + + p := planner{ + cap: 2, + min: 1, + max: 4, + ttu: time.Hour, + client: client, + servers: store, + } + + err := p.Plan(context.TODO()) + if err != nil { + t.Error(err) + } +} + +func TestPlan_ShutdownIdle(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + // x3 capacity + servers := []*autoscaler.Server{ + {Name: "server1", Capacity: 2, Created: 1, State: autoscaler.StateRunning}, + {Name: "server2", Capacity: 2, Created: 2, State: autoscaler.StateRunning}, + {Name: "server3", Capacity: 2, Created: 3, State: autoscaler.StateRunning}, + } + + // x0 running builds + // x0 pending builds + builds := []*drone.Activity{} + + store := mocks.NewMockServerStore(controller) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + store.EXPECT().List(gomock.Any()).Return(servers, nil) + store.EXPECT().Update(gomock.Any(), servers[2]).Return(nil) + store.EXPECT().Update(gomock.Any(), servers[1]).Return(nil) + + client := mocks.NewMockClient(controller) + client.EXPECT().BuildQueue().Return(builds, nil) + client.EXPECT().BuildQueue().Return(builds, nil) + + p := planner{ + cap: 2, + min: 1, + max: 4, + client: client, + servers: store, + } + + err := p.Plan(context.TODO()) + if err != nil { + t.Error(err) + } +} + +// func TestListBusy(t *testing.T) { +// controller := gomock.NewController(t) +// defer controller.Finish() + +// client := mocks.NewMockClient(controller) +// client.EXPECT().Build("octocat", "hello-world", 1).Return(&drone.Build{ +// Procs: []*drone.Proc{ +// {PID: 1, Machine: "machine1"}, +// {PID: 2, Machine: "machine2"}, +// }, +// }, nil) +// client.EXPECT().BuildQueue().Return([]*drone.Activity{ +// {Status: drone.StatusPending}, +// {Status: drone.StatusRunning, Owner: "octocat", Name: "hello-world", Number: 1}, +// }, nil) + +// scaler := Scaler{Client: client} +// busy, err := scaler.listBusy(context.TODO()) +// if err != nil { +// t.Error(err) +// return +// } +// if got, want := len(busy), 2; got != want { +// t.Errorf("Want busy server count %d, got %d", want, got) +// } +// if _, ok := busy["machine1"]; !ok { +// t.Errorf("Expected server not in busy list") +// } +// if _, ok := busy["machine2"]; !ok { +// t.Errorf("Expected server not in busy list") +// } +// } + +// func TestCapacity(t *testing.T) { +// controller := gomock.NewController(t) +// defer controller.Finish() + +// servers := []*autoscaler.Server{ +// {Name: "server1", Capacity: 4}, +// {Name: "server2", Capacity: 3}, +// {Name: "server3", Capacity: 2}, +// {Name: "server4", Capacity: 1}, +// } + +// store := mocks.NewMockServerStore(controller) +// store.EXPECT().List(gomock.Any()).Return(servers, nil) + +// scaler := Scaler{Servers: store} +// capacity, count, err := scaler.capacity(context.TODO()) +// if err != nil { +// t.Error(err) +// return +// } +// if got, want := capacity, 10; got != want { +// t.Errorf("Want capacity count %d, got %d", want, got) +// } +// if got, want := count, 4; got != want { +// t.Errorf("Want server count %d, got %d", want, got) +// } +// } + +// func TestCount(t *testing.T) { +// controller := gomock.NewController(t) +// defer controller.Finish() + +// client := mocks.NewMockClient(controller) +// client.EXPECT().BuildQueue().Return([]*drone.Activity{ +// {Status: drone.StatusPending}, +// {Status: drone.StatusPending}, +// {Status: drone.StatusPending}, +// {Status: drone.StatusRunning}, +// {Status: drone.StatusRunning}, +// }, nil) + +// scaler := Scaler{Client: client} +// pending, running, err := scaler.count(context.TODO()) +// if err != nil { +// t.Error(err) +// return +// } +// if got, want := pending, 3; got != want { +// t.Errorf("Want pending count %d, got %d", want, got) +// } +// if got, want := running, 2; got != want { +// t.Errorf("Want running count %d, got %d", want, got) +// } +// } diff --git a/engine/sort.go b/engine/sort.go new file mode 100644 index 00000000..7b53e4a0 --- /dev/null +++ b/engine/sort.go @@ -0,0 +1,14 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import "github.com/drone/autoscaler" + +// byCreated sorts the server list by created date. +type byCreated []*autoscaler.Server + +func (a byCreated) Len() int { return len(a) } +func (a byCreated) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byCreated) Less(i, j int) bool { return a[i].Created < a[j].Created } diff --git a/engine/sort_test.go b/engine/sort_test.go new file mode 100644 index 00000000..e4b140f7 --- /dev/null +++ b/engine/sort_test.go @@ -0,0 +1,30 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package engine + +import ( + "sort" + "testing" + + "github.com/drone/autoscaler" +) + +func TestSortByCreated(t *testing.T) { + servers := []*autoscaler.Server{ + {Created: 4, Name: "fourth"}, + {Created: 2, Name: "second"}, + {Created: 3, Name: "third"}, + {Created: 5, Name: "fifth"}, + {Created: 1, Name: "first"}, + } + + sort.Sort(byCreated(servers)) + + for i, server := range servers { + if server.Created != int64(i+1) { + t.Errorf("Invalid sort order %d for %q", i, server.Name) + } + } +} diff --git a/metrics/pool_max.go b/metrics/pool_max.go deleted file mode 100644 index 1be32258..00000000 --- a/metrics/pool_max.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package metrics - -import ( - "github.com/drone/autoscaler/config" - "github.com/prometheus/client_golang/prometheus" -) - -// MaxPool exposes the min pool metric. -func MaxPool(config config.Config) { - prometheus.MustRegister( - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "drone_server_max_pool", - Help: "Maximum number of active servers.", - }, func() float64 { - return float64(config.Pool.Max) - }), - ) -} diff --git a/metrics/pool_max_test.go b/metrics/pool_max_test.go deleted file mode 100644 index a6323de8..00000000 --- a/metrics/pool_max_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package metrics - -import ( - "testing" - - "github.com/drone/autoscaler/config" - "github.com/prometheus/client_golang/prometheus" -) - -func TestMaxPool(t *testing.T) { - // restore the default prometheus registerer - // when the unit test is complete. - snapshot := prometheus.DefaultRegisterer - defer func() { - prometheus.DefaultRegisterer = snapshot - }() - - // creates a blank registry - registry := prometheus.NewRegistry() - prometheus.DefaultRegisterer = registry - - conf := config.Config{} - conf.Pool.Max = 10 - MaxPool(conf) - - metrics, err := registry.Gather() - if err != nil { - t.Error(err) - return - } - if want, got := len(metrics), 1; want != got { - t.Errorf("Expect registered metric") - return - } - metric := metrics[0] - if want, got := metric.GetName(), "drone_server_max_pool"; want != got { - t.Errorf("Expect metric name %s, got %s", want, got) - } - if want, got := metric.Metric[0].Gauge.GetValue(), float64(10); want != got { - t.Errorf("Expect metric value %f, got %f", want, got) - } -} diff --git a/metrics/pool_min.go b/metrics/pool_min.go deleted file mode 100644 index 05585ceb..00000000 --- a/metrics/pool_min.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package metrics - -import ( - "github.com/drone/autoscaler/config" - "github.com/prometheus/client_golang/prometheus" -) - -// MinPool exposes the min pool metric. -func MinPool(config config.Config) { - prometheus.MustRegister( - prometheus.NewGaugeFunc(prometheus.GaugeOpts{ - Name: "drone_server_min_pool", - Help: "Minimum number of active servers.", - }, func() float64 { - return float64(config.Pool.Min) - }), - ) -} diff --git a/metrics/pool_min_test.go b/metrics/pool_min_test.go deleted file mode 100644 index 809ce153..00000000 --- a/metrics/pool_min_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package metrics - -import ( - "testing" - - "github.com/drone/autoscaler/config" - "github.com/prometheus/client_golang/prometheus" -) - -func TestMinPool(t *testing.T) { - // restore the default prometheus registerer - // when the unit test is complete. - snapshot := prometheus.DefaultRegisterer - defer func() { - prometheus.DefaultRegisterer = snapshot - }() - - // creates a blank registry - registry := prometheus.NewRegistry() - prometheus.DefaultRegisterer = registry - - conf := config.Config{} - conf.Pool.Min = 5 - MinPool(conf) - - metrics, err := registry.Gather() - if err != nil { - t.Error(err) - return - } - if want, got := len(metrics), 1; want != got { - t.Errorf("Expect registered metric") - return - } - metric := metrics[0] - if want, got := metric.GetName(), "drone_server_min_pool"; want != got { - t.Errorf("Expect metric name %s, got %s", want, got) - } - if want, got := metric.Metric[0].Gauge.GetValue(), float64(5); want != got { - t.Errorf("Expect metric value %f, got %f", want, got) - } -} diff --git a/metrics/server_capacity.go b/metrics/server_capacity.go index 0d18209c..62c67a4b 100644 --- a/metrics/server_capacity.go +++ b/metrics/server_capacity.go @@ -17,7 +17,7 @@ func ServerCapacity(store autoscaler.ServerStore) autoscaler.ServerStore { Help: "Total capacity of active servers.", }, func() float64 { var capacity int - servers, _ := store.List(noContext) + servers, _ := store.ListState(noContext, autoscaler.StateRunning) for _, server := range servers { capacity += server.Capacity } diff --git a/metrics/server_capacity_test.go b/metrics/server_capacity_test.go index 6025487c..2beba3ba 100644 --- a/metrics/server_capacity_test.go +++ b/metrics/server_capacity_test.go @@ -37,7 +37,7 @@ func TestServerCapacity(t *testing.T) { } store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) + store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil) ServerCapacity(store) metrics, err := registry.Gather() diff --git a/metrics/server_count.go b/metrics/server_count.go index 8c812219..67fe6772 100644 --- a/metrics/server_count.go +++ b/metrics/server_count.go @@ -16,7 +16,7 @@ func ServerCount(store autoscaler.ServerStore) autoscaler.ServerStore { Name: "drone_server_count", Help: "Total number of active servers.", }, func() float64 { - servers, _ := store.List(noContext) + servers, _ := store.ListState(noContext, autoscaler.StateRunning) return float64(len(servers)) }), ) diff --git a/metrics/server_count_test.go b/metrics/server_count_test.go index 08e6a795..01757e78 100644 --- a/metrics/server_count_test.go +++ b/metrics/server_count_test.go @@ -36,7 +36,7 @@ func TestServerCount(t *testing.T) { } store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) + store.EXPECT().ListState(gomock.Any(), autoscaler.StateRunning).Return(servers, nil) ServerCount(store) metrics, err := registry.Gather() diff --git a/metrics/server_create.go b/metrics/server_create.go index fbe3ebe1..c9725616 100644 --- a/metrics/server_create.go +++ b/metrics/server_create.go @@ -37,12 +37,12 @@ type providerWrapCreate struct { errors prometheus.Counter } -func (p *providerWrapCreate) Create(ctx context.Context, opts *autoscaler.ServerOpts) (*autoscaler.Server, error) { - server, err := p.Provider.Create(ctx, opts) +func (p *providerWrapCreate) Create(ctx context.Context, opts autoscaler.InstanceCreateOpts) (*autoscaler.Instance, error) { + instance, err := p.Provider.Create(ctx, opts) if err == nil { p.created.Add(1) } else { p.errors.Add(1) } - return server, err + return instance, err } diff --git a/metrics/server_create_test.go b/metrics/server_create_test.go index 02f6f427..7c83b334 100644 --- a/metrics/server_create_test.go +++ b/metrics/server_create_test.go @@ -8,8 +8,6 @@ import ( "errors" "testing" - "github.com/kr/pretty" - "github.com/drone/autoscaler" "github.com/drone/autoscaler/mocks" "github.com/golang/mock/gomock" @@ -31,22 +29,21 @@ func TestServerCreate(t *testing.T) { registry := prometheus.NewRegistry() prometheus.DefaultRegisterer = registry - opts := &autoscaler.ServerOpts{} - server := &autoscaler.Server{Name: "server1", Capacity: 1} + opts := autoscaler.InstanceCreateOpts{Name: "server1"} + instance := &autoscaler.Instance{} provider := mocks.NewMockProvider(controller) - provider.EXPECT().Create(gomock.Any(), opts).Times(3).Return(server, nil) + provider.EXPECT().Create(gomock.Any(), opts).Times(3).Return(instance, nil) provider.EXPECT().Create(gomock.Any(), opts).Return(nil, errors.New("error")) providerInst := ServerCreate(provider) for i := 0; i < 3; i++ { - result, err := providerInst.Create(noContext, opts) + res, err := providerInst.Create(noContext, opts) if err != nil { t.Error(err) } - if got, want := result, server; got != want { - t.Errorf("Expect server returned from provider, CALL %d", i) - pretty.Ldiff(t, got, want) + if res != instance { + t.Errorf("Expect instance returned") } } _, err := providerInst.Create(noContext, opts) diff --git a/metrics/server_delete.go b/metrics/server_delete.go index 8296b9c1..169ce061 100644 --- a/metrics/server_delete.go +++ b/metrics/server_delete.go @@ -37,8 +37,8 @@ type providerWrapDestroy struct { errors prometheus.Counter } -func (p *providerWrapDestroy) Destroy(ctx context.Context, server *autoscaler.Server) error { - err := p.Provider.Destroy(ctx, server) +func (p *providerWrapDestroy) Destroy(ctx context.Context, instance *autoscaler.Instance) error { + err := p.Provider.Destroy(ctx, instance) if err == nil { p.created.Add(1) } else { diff --git a/metrics/server_delete_test.go b/metrics/server_delete_test.go index a0c8eded..0890df26 100644 --- a/metrics/server_delete_test.go +++ b/metrics/server_delete_test.go @@ -29,20 +29,20 @@ func TestServerDelete(t *testing.T) { registry := prometheus.NewRegistry() prometheus.DefaultRegisterer = registry - server := &autoscaler.Server{Name: "server1", Capacity: 1} + instance := &autoscaler.Instance{Name: "server1"} provider := mocks.NewMockProvider(controller) - provider.EXPECT().Destroy(noContext, server).Times(3).Return(nil) - provider.EXPECT().Destroy(noContext, server).Return(errors.New("error")) + provider.EXPECT().Destroy(noContext, instance).Times(3).Return(nil) + provider.EXPECT().Destroy(noContext, instance).Return(errors.New("error")) providerInst := ServerDelete(provider) for i := 0; i < 3; i++ { - err := providerInst.Destroy(noContext, server) + err := providerInst.Destroy(noContext, instance) if err != nil { t.Error(err) } } - err := providerInst.Destroy(noContext, server) + err := providerInst.Destroy(noContext, instance) if err == nil { t.Errorf("Expect error returned from provider") } diff --git a/mocks/mocks_gen_drone.go b/mocks/mock_drone.go similarity index 87% rename from mocks/mocks_gen_drone.go rename to mocks/mock_drone.go index 515cf012..7ef4f795 100644 --- a/mocks/mocks_gen_drone.go +++ b/mocks/mock_drone.go @@ -34,6 +34,43 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { return m.recorder } +// AutoscalePause mocks base method +func (m *MockClient) AutoscalePause() error { + ret := m.ctrl.Call(m, "AutoscalePause") + ret0, _ := ret[0].(error) + return ret0 +} + +// AutoscalePause indicates an expected call of AutoscalePause +func (mr *MockClientMockRecorder) AutoscalePause() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoscalePause", reflect.TypeOf((*MockClient)(nil).AutoscalePause)) +} + +// AutoscaleResume mocks base method +func (m *MockClient) AutoscaleResume() error { + ret := m.ctrl.Call(m, "AutoscaleResume") + ret0, _ := ret[0].(error) + return ret0 +} + +// AutoscaleResume indicates an expected call of AutoscaleResume +func (mr *MockClientMockRecorder) AutoscaleResume() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoscaleResume", reflect.TypeOf((*MockClient)(nil).AutoscaleResume)) +} + +// AutoscaleVersion mocks base method +func (m *MockClient) AutoscaleVersion() (*drone.Version, error) { + ret := m.ctrl.Call(m, "AutoscaleVersion") + ret0, _ := ret[0].(*drone.Version) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AutoscaleVersion indicates an expected call of AutoscaleVersion +func (mr *MockClientMockRecorder) AutoscaleVersion() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AutoscaleVersion", reflect.TypeOf((*MockClient)(nil).AutoscaleVersion)) +} + // Build mocks base method func (m *MockClient) Build(arg0, arg1 string, arg2 int) (*drone.Build, error) { ret := m.ctrl.Call(m, "Build", arg0, arg1, arg2) @@ -162,6 +199,18 @@ func (mr *MockClientMockRecorder) Deploy(arg0, arg1, arg2, arg3, arg4 interface{ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Deploy", reflect.TypeOf((*MockClient)(nil).Deploy), arg0, arg1, arg2, arg3, arg4) } +// LogsPurge mocks base method +func (m *MockClient) LogsPurge(arg0, arg1 string, arg2 int) error { + ret := m.ctrl.Call(m, "LogsPurge", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// LogsPurge indicates an expected call of LogsPurge +func (mr *MockClientMockRecorder) LogsPurge(arg0, arg1, arg2 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LogsPurge", reflect.TypeOf((*MockClient)(nil).LogsPurge), arg0, arg1, arg2) +} + // Registry mocks base method func (m *MockClient) Registry(arg0, arg1, arg2 string) (*drone.Registry, error) { ret := m.ctrl.Call(m, "Registry", arg0, arg1, arg2) @@ -417,6 +466,31 @@ func (mr *MockClientMockRecorder) Server(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Server", reflect.TypeOf((*MockClient)(nil).Server), arg0) } +// ServerCreate mocks base method +func (m *MockClient) ServerCreate() (*drone.Server, error) { + ret := m.ctrl.Call(m, "ServerCreate") + ret0, _ := ret[0].(*drone.Server) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ServerCreate indicates an expected call of ServerCreate +func (mr *MockClientMockRecorder) ServerCreate() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServerCreate", reflect.TypeOf((*MockClient)(nil).ServerCreate)) +} + +// ServerDelete mocks base method +func (m *MockClient) ServerDelete(arg0 string) error { + ret := m.ctrl.Call(m, "ServerDelete", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// ServerDelete indicates an expected call of ServerDelete +func (mr *MockClientMockRecorder) ServerDelete(arg0 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ServerDelete", reflect.TypeOf((*MockClient)(nil).ServerDelete), arg0) +} + // ServerList mocks base method func (m *MockClient) ServerList() ([]*drone.Server, error) { ret := m.ctrl.Call(m, "ServerList") diff --git a/mocks/mock_engine.go b/mocks/mock_engine.go new file mode 100644 index 00000000..69d2ce9e --- /dev/null +++ b/mocks/mock_engine.go @@ -0,0 +1,76 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/drone/autoscaler (interfaces: Engine) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockEngine is a mock of Engine interface +type MockEngine struct { + ctrl *gomock.Controller + recorder *MockEngineMockRecorder +} + +// MockEngineMockRecorder is the mock recorder for MockEngine +type MockEngineMockRecorder struct { + mock *MockEngine +} + +// NewMockEngine creates a new mock instance +func NewMockEngine(ctrl *gomock.Controller) *MockEngine { + mock := &MockEngine{ctrl: ctrl} + mock.recorder = &MockEngineMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockEngine) EXPECT() *MockEngineMockRecorder { + return m.recorder +} + +// Pause mocks base method +func (m *MockEngine) Pause() { + m.ctrl.Call(m, "Pause") +} + +// Pause indicates an expected call of Pause +func (mr *MockEngineMockRecorder) Pause() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pause", reflect.TypeOf((*MockEngine)(nil).Pause)) +} + +// Paused mocks base method +func (m *MockEngine) Paused() bool { + ret := m.ctrl.Call(m, "Paused") + ret0, _ := ret[0].(bool) + return ret0 +} + +// Paused indicates an expected call of Paused +func (mr *MockEngineMockRecorder) Paused() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Paused", reflect.TypeOf((*MockEngine)(nil).Paused)) +} + +// Resume mocks base method +func (m *MockEngine) Resume() { + m.ctrl.Call(m, "Resume") +} + +// Resume indicates an expected call of Resume +func (mr *MockEngineMockRecorder) Resume() *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resume", reflect.TypeOf((*MockEngine)(nil).Resume)) +} + +// Start mocks base method +func (m *MockEngine) Start(arg0 context.Context) { + m.ctrl.Call(m, "Start", arg0) +} + +// Start indicates an expected call of Start +func (mr *MockEngineMockRecorder) Start(arg0 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockEngine)(nil).Start), arg0) +} diff --git a/mocks/mock_provider.go b/mocks/mock_provider.go new file mode 100644 index 00000000..a0fe849c --- /dev/null +++ b/mocks/mock_provider.go @@ -0,0 +1,85 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/drone/autoscaler (interfaces: Provider) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + autoscaler "github.com/drone/autoscaler" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockProvider is a mock of Provider interface +type MockProvider struct { + ctrl *gomock.Controller + recorder *MockProviderMockRecorder +} + +// MockProviderMockRecorder is the mock recorder for MockProvider +type MockProviderMockRecorder struct { + mock *MockProvider +} + +// NewMockProvider creates a new mock instance +func NewMockProvider(ctrl *gomock.Controller) *MockProvider { + mock := &MockProvider{ctrl: ctrl} + mock.recorder = &MockProviderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockProvider) EXPECT() *MockProviderMockRecorder { + return m.recorder +} + +// Create mocks base method +func (m *MockProvider) Create(arg0 context.Context, arg1 autoscaler.InstanceCreateOpts) (*autoscaler.Instance, error) { + ret := m.ctrl.Call(m, "Create", arg0, arg1) + ret0, _ := ret[0].(*autoscaler.Instance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Create indicates an expected call of Create +func (mr *MockProviderMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockProvider)(nil).Create), arg0, arg1) +} + +// Destroy mocks base method +func (m *MockProvider) Destroy(arg0 context.Context, arg1 *autoscaler.Instance) error { + ret := m.ctrl.Call(m, "Destroy", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Destroy indicates an expected call of Destroy +func (mr *MockProviderMockRecorder) Destroy(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Destroy", reflect.TypeOf((*MockProvider)(nil).Destroy), arg0, arg1) +} + +// Execute mocks base method +func (m *MockProvider) Execute(arg0 context.Context, arg1 *autoscaler.Instance, arg2 string) ([]byte, error) { + ret := m.ctrl.Call(m, "Execute", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Execute indicates an expected call of Execute +func (mr *MockProviderMockRecorder) Execute(arg0, arg1, arg2 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Execute", reflect.TypeOf((*MockProvider)(nil).Execute), arg0, arg1, arg2) +} + +// Ping mocks base method +func (m *MockProvider) Ping(arg0 context.Context, arg1 *autoscaler.Instance) error { + ret := m.ctrl.Call(m, "Ping", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Ping indicates an expected call of Ping +func (mr *MockProviderMockRecorder) Ping(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ping", reflect.TypeOf((*MockProvider)(nil).Ping), arg0, arg1) +} diff --git a/mocks/mock_server.go b/mocks/mock_server.go new file mode 100644 index 00000000..bb82d87c --- /dev/null +++ b/mocks/mock_server.go @@ -0,0 +1,122 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/drone/autoscaler (interfaces: ServerStore) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + autoscaler "github.com/drone/autoscaler" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockServerStore is a mock of ServerStore interface +type MockServerStore struct { + ctrl *gomock.Controller + recorder *MockServerStoreMockRecorder +} + +// MockServerStoreMockRecorder is the mock recorder for MockServerStore +type MockServerStoreMockRecorder struct { + mock *MockServerStore +} + +// NewMockServerStore creates a new mock instance +func NewMockServerStore(ctrl *gomock.Controller) *MockServerStore { + mock := &MockServerStore{ctrl: ctrl} + mock.recorder = &MockServerStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockServerStore) EXPECT() *MockServerStoreMockRecorder { + return m.recorder +} + +// Create mocks base method +func (m *MockServerStore) Create(arg0 context.Context, arg1 *autoscaler.Server) error { + ret := m.ctrl.Call(m, "Create", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create +func (mr *MockServerStoreMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockServerStore)(nil).Create), arg0, arg1) +} + +// Delete mocks base method +func (m *MockServerStore) Delete(arg0 context.Context, arg1 *autoscaler.Server) error { + ret := m.ctrl.Call(m, "Delete", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete +func (mr *MockServerStoreMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockServerStore)(nil).Delete), arg0, arg1) +} + +// Find mocks base method +func (m *MockServerStore) Find(arg0 context.Context, arg1 string) (*autoscaler.Server, error) { + ret := m.ctrl.Call(m, "Find", arg0, arg1) + ret0, _ := ret[0].(*autoscaler.Server) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Find indicates an expected call of Find +func (mr *MockServerStoreMockRecorder) Find(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockServerStore)(nil).Find), arg0, arg1) +} + +// List mocks base method +func (m *MockServerStore) List(arg0 context.Context) ([]*autoscaler.Server, error) { + ret := m.ctrl.Call(m, "List", arg0) + ret0, _ := ret[0].([]*autoscaler.Server) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// List indicates an expected call of List +func (mr *MockServerStoreMockRecorder) List(arg0 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockServerStore)(nil).List), arg0) +} + +// ListState mocks base method +func (m *MockServerStore) ListState(arg0 context.Context, arg1 autoscaler.ServerState) ([]*autoscaler.Server, error) { + ret := m.ctrl.Call(m, "ListState", arg0, arg1) + ret0, _ := ret[0].([]*autoscaler.Server) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListState indicates an expected call of ListState +func (mr *MockServerStoreMockRecorder) ListState(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListState", reflect.TypeOf((*MockServerStore)(nil).ListState), arg0, arg1) +} + +// Purge mocks base method +func (m *MockServerStore) Purge(arg0 context.Context, arg1 int64) error { + ret := m.ctrl.Call(m, "Purge", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Purge indicates an expected call of Purge +func (mr *MockServerStoreMockRecorder) Purge(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Purge", reflect.TypeOf((*MockServerStore)(nil).Purge), arg0, arg1) +} + +// Update mocks base method +func (m *MockServerStore) Update(arg0 context.Context, arg1 *autoscaler.Server) error { + ret := m.ctrl.Call(m, "Update", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update +func (mr *MockServerStoreMockRecorder) Update(arg0, arg1 interface{}) *gomock.Call { + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockServerStore)(nil).Update), arg0, arg1) +} diff --git a/mocks/mocks.go b/mocks/mocks.go index 0748d1f1..870dcdd0 100644 --- a/mocks/mocks.go +++ b/mocks/mocks.go @@ -4,5 +4,7 @@ package mocks -//go:generate mockgen -package=mocks -destination=mocks_gen.go github.com/drone/autoscaler Scaler,ServerStore,Provider -//go:generate mockgen -package=mocks -destination=mocks_gen_drone.go github.com/drone/drone-go/drone Client +//go:generate mockgen -package=mocks -destination=mock_engine.go github.com/drone/autoscaler Engine +//go:generate mockgen -package=mocks -destination=mock_server.go github.com/drone/autoscaler ServerStore +//go:generate mockgen -package=mocks -destination=mock_provider.go github.com/drone/autoscaler Provider +//go:generate mockgen -package=mocks -destination=mock_drone.go github.com/drone/drone-go/drone Client diff --git a/mocks/mocks_gen.go b/mocks/mocks_gen.go deleted file mode 100644 index b59becb3..00000000 --- a/mocks/mocks_gen.go +++ /dev/null @@ -1,212 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/drone/autoscaler (interfaces: Scaler,ServerStore,Provider) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - autoscaler "github.com/drone/autoscaler" - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockScaler is a mock of Scaler interface -type MockScaler struct { - ctrl *gomock.Controller - recorder *MockScalerMockRecorder -} - -// MockScalerMockRecorder is the mock recorder for MockScaler -type MockScalerMockRecorder struct { - mock *MockScaler -} - -// NewMockScaler creates a new mock instance -func NewMockScaler(ctrl *gomock.Controller) *MockScaler { - mock := &MockScaler{ctrl: ctrl} - mock.recorder = &MockScalerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockScaler) EXPECT() *MockScalerMockRecorder { - return m.recorder -} - -// Pause mocks base method -func (m *MockScaler) Pause() { - m.ctrl.Call(m, "Pause") -} - -// Pause indicates an expected call of Pause -func (mr *MockScalerMockRecorder) Pause() *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Pause", reflect.TypeOf((*MockScaler)(nil).Pause)) -} - -// Paused mocks base method -func (m *MockScaler) Paused() bool { - ret := m.ctrl.Call(m, "Paused") - ret0, _ := ret[0].(bool) - return ret0 -} - -// Paused indicates an expected call of Paused -func (mr *MockScalerMockRecorder) Paused() *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Paused", reflect.TypeOf((*MockScaler)(nil).Paused)) -} - -// Resume mocks base method -func (m *MockScaler) Resume() { - m.ctrl.Call(m, "Resume") -} - -// Resume indicates an expected call of Resume -func (mr *MockScalerMockRecorder) Resume() *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Resume", reflect.TypeOf((*MockScaler)(nil).Resume)) -} - -// Scale mocks base method -func (m *MockScaler) Scale(arg0 context.Context) error { - ret := m.ctrl.Call(m, "Scale", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Scale indicates an expected call of Scale -func (mr *MockScalerMockRecorder) Scale(arg0 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Scale", reflect.TypeOf((*MockScaler)(nil).Scale), arg0) -} - -// MockServerStore is a mock of ServerStore interface -type MockServerStore struct { - ctrl *gomock.Controller - recorder *MockServerStoreMockRecorder -} - -// MockServerStoreMockRecorder is the mock recorder for MockServerStore -type MockServerStoreMockRecorder struct { - mock *MockServerStore -} - -// NewMockServerStore creates a new mock instance -func NewMockServerStore(ctrl *gomock.Controller) *MockServerStore { - mock := &MockServerStore{ctrl: ctrl} - mock.recorder = &MockServerStoreMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockServerStore) EXPECT() *MockServerStoreMockRecorder { - return m.recorder -} - -// Create mocks base method -func (m *MockServerStore) Create(arg0 context.Context, arg1 *autoscaler.Server) error { - ret := m.ctrl.Call(m, "Create", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create -func (mr *MockServerStoreMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockServerStore)(nil).Create), arg0, arg1) -} - -// Delete mocks base method -func (m *MockServerStore) Delete(arg0 context.Context, arg1 *autoscaler.Server) error { - ret := m.ctrl.Call(m, "Delete", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete -func (mr *MockServerStoreMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockServerStore)(nil).Delete), arg0, arg1) -} - -// Find mocks base method -func (m *MockServerStore) Find(arg0 context.Context, arg1 string) (*autoscaler.Server, error) { - ret := m.ctrl.Call(m, "Find", arg0, arg1) - ret0, _ := ret[0].(*autoscaler.Server) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Find indicates an expected call of Find -func (mr *MockServerStoreMockRecorder) Find(arg0, arg1 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Find", reflect.TypeOf((*MockServerStore)(nil).Find), arg0, arg1) -} - -// List mocks base method -func (m *MockServerStore) List(arg0 context.Context) ([]*autoscaler.Server, error) { - ret := m.ctrl.Call(m, "List", arg0) - ret0, _ := ret[0].([]*autoscaler.Server) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// List indicates an expected call of List -func (mr *MockServerStoreMockRecorder) List(arg0 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockServerStore)(nil).List), arg0) -} - -// Update mocks base method -func (m *MockServerStore) Update(arg0 context.Context, arg1 *autoscaler.Server) error { - ret := m.ctrl.Call(m, "Update", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Update indicates an expected call of Update -func (mr *MockServerStoreMockRecorder) Update(arg0, arg1 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockServerStore)(nil).Update), arg0, arg1) -} - -// MockProvider is a mock of Provider interface -type MockProvider struct { - ctrl *gomock.Controller - recorder *MockProviderMockRecorder -} - -// MockProviderMockRecorder is the mock recorder for MockProvider -type MockProviderMockRecorder struct { - mock *MockProvider -} - -// NewMockProvider creates a new mock instance -func NewMockProvider(ctrl *gomock.Controller) *MockProvider { - mock := &MockProvider{ctrl: ctrl} - mock.recorder = &MockProviderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockProvider) EXPECT() *MockProviderMockRecorder { - return m.recorder -} - -// Create mocks base method -func (m *MockProvider) Create(arg0 context.Context, arg1 *autoscaler.ServerOpts) (*autoscaler.Server, error) { - ret := m.ctrl.Call(m, "Create", arg0, arg1) - ret0, _ := ret[0].(*autoscaler.Server) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Create indicates an expected call of Create -func (mr *MockProviderMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockProvider)(nil).Create), arg0, arg1) -} - -// Destroy mocks base method -func (m *MockProvider) Destroy(arg0 context.Context, arg1 *autoscaler.Server) error { - ret := m.ctrl.Call(m, "Destroy", arg0, arg1) - ret0, _ := ret[0].(error) - return ret0 -} - -// Destroy indicates an expected call of Destroy -func (mr *MockProviderMockRecorder) Destroy(arg0, arg1 interface{}) *gomock.Call { - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Destroy", reflect.TypeOf((*MockProvider)(nil).Destroy), arg0, arg1) -} diff --git a/provider.go b/provider.go index 0d45e029..576997f5 100644 --- a/provider.go +++ b/provider.go @@ -7,23 +7,62 @@ package autoscaler import "context" // ProviderType specifies the hosting provider. -type ProviderType int +type ProviderType string // Provider type enumeration. const ( - ProviderUnknown ProviderType = iota - ProviderAmazon - ProviderAzure - ProviderDigitalOcean - ProviderGoogle + ProviderAmazon = ProviderType("amazon") + ProviderAzure = ProviderType("azure") + ProviderDigitalOcean = ProviderType("digitalocean") + ProviderGoogle = ProviderType("google") + ProviderLinode = ProviderType("linode") + ProviderOpenStack = ProviderType("openstack") + ProviderScaleway = ProviderType("scaleway") + ProviderVultr = ProviderType("vultr") + ProviderHetznerCloud = ProviderType("hetznercloud") ) -// A Provider represents a hosting provider, such as Digital Ocean -// and is responsible for server management. +// A Provider represents a hosting provider, such as +// Digital Ocean and is responsible for server management. type Provider interface { // Create creates a new server. - Create(context.Context, *ServerOpts) (*Server, error) - + Create(context.Context, InstanceCreateOpts) (*Instance, error) // Destroy destroys an existing server. - Destroy(context.Context, *Server) error + Destroy(context.Context, *Instance) error + // Execute executes a command on the remote server and + // returns the combined terminal output. + Execute(context.Context, *Instance, string) ([]byte, error) + // Ping pings the remote server. + Ping(context.Context, *Instance) error +} + +// An Instance represents a server instance +// (e.g Digital Ocean Droplet). +type Instance struct { + Provider ProviderType + ID string + Name string + Address string + Region string + Image string + Size string + Secret string +} + +// InstanceCreateOpts define soptional instructions for +// creating server instances. +type InstanceCreateOpts struct { + Name string +} + +// InstanceError snapshots an error creating an instance +// with server logs. +type InstanceError struct { + Err error + Logs []byte +} + +// Error implements the error interface. +func (e *InstanceError) Error() string { + return e.Err.Error() } diff --git a/scaler.go b/scaler.go deleted file mode 100644 index 87d3050c..00000000 --- a/scaler.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package autoscaler - -import ( - "context" -) - -// A Scaler implements an algorithm to automatically scale up -// or scale down the available pool of servers. -type Scaler interface { - Pause() - Paused() bool - Resume() - - Scale(context.Context) error -} diff --git a/scaler/loop.go b/scaler/loop.go deleted file mode 100644 index ca090aa5..00000000 --- a/scaler/loop.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package scaler - -import ( - "context" - "time" - - "github.com/drone/autoscaler" -) - -// Start executes the synchronizer in a loop. -func Start(ctx context.Context, scaler autoscaler.Scaler, duration time.Duration) error { - for { - select { - case <-time.After(duration): - if !scaler.Paused() { - scaler.Scale(ctx) - } - case <-ctx.Done(): - return ctx.Err() - } - } -} diff --git a/scaler/scaler.go b/scaler/scaler.go deleted file mode 100644 index 839e4a24..00000000 --- a/scaler/scaler.go +++ /dev/null @@ -1,300 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package scaler - -import ( - "context" - "math" - "sort" - "sync" - "time" - - "github.com/drone/autoscaler" - "github.com/drone/autoscaler/config" - "github.com/drone/drone-go/drone" - - "github.com/rs/zerolog/log" - "golang.org/x/sync/errgroup" -) - -// Scaler represents the built-in auto-scaler. -type Scaler struct { - mu sync.Mutex - - paused bool - Client drone.Client - Config config.Config - Servers autoscaler.ServerStore - Provider autoscaler.Provider -} - -// Pause paueses the scaler. -func (s *Scaler) Pause() { - s.mu.Lock() - s.paused = true - s.mu.Unlock() -} - -// Paused returns true if scaling is paused. -func (s *Scaler) Paused() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.paused -} - -// Resume resumes the scaler. -func (s *Scaler) Resume() { - s.mu.Lock() - s.paused = false - s.mu.Unlock() -} - -// Scale execute the autoscaling algorithm. -func (s *Scaler) Scale(ctx context.Context) error { - logger := log.Ctx(ctx) - - pending, running, err := s.count(ctx) - if err != nil { - logger.Error().Err(err). - Msg("Error fetching queue details") - return err - } - capacity, servers, err := s.capacity(ctx) - if err != nil { - logger.Error().Err(err). - Msg("Error calculating server capacity") - return err - } - - ctx = logger.With(). - Int("min-pool", s.Config.Pool.Min). - Int("max-pool", s.Config.Pool.Max). - Int("capacity", capacity). - Int("pending", pending). - Int("running", running). - Logger().WithContext(ctx) - - free := capacity - running - need := requiredCapacity(pending, free, s.Config.Agent.Concurrency) - if need > 0 { - // do not increase the pool beyond the max size. - if servers+need >= s.Config.Pool.Max { - need = s.Config.Pool.Max - servers - } - if need < 0 { - need = 0 - } - return s.provision(ctx, need) - } - return s.collect(ctx) -} - -// provision provisions n new servers. -func (s *Scaler) provision(ctx context.Context, n int) error { - logger := log.Ctx(ctx) - logger.Debug(). - Int("create", n). - Msgf("create %d instances", n) - - var g errgroup.Group - for i := 0; i < n; i++ { - g.Go(func() error { - return s.provisionOne(ctx) - }) - } - return g.Wait() -} - -// provisionOne provisions a new server. -func (s *Scaler) provisionOne(ctx context.Context) error { - opts := autoscaler.NewServerOpts("agent", s.Config.Agent.Concurrency) - server, err := s.Provider.Create(ctx, opts) - if err != nil { - log.Ctx(ctx).Error().Err(err). - Msg("failed to provision instance") - return err - } - return s.Servers.Create(ctx, server) -} - -// collect garbage collects servers that can be released -// due to inactivity. -func (s *Scaler) collect(ctx context.Context) error { - logger := log.Ctx(ctx) - - servers, err := s.Servers.List(ctx) - if err != nil { - logger.Error().Err(err). - Msg("failed to fetch server list") - return err - } - - // do not garbage collect any servers if the pool - // is at or below the minimum limit. - if len(servers) <= s.Config.Pool.Min { - logger.Debug(). - Msg("minimum server capacity") - return nil - } - - // sort by created date. older servers are retired first. - sort.Sort(sort.Reverse(autoscaler.ByCreated(servers))) - - busy, err := s.listBusy(ctx) - if err != nil { - logger.Error().Err(err). - Msg("failed to fetch busy list") - return err - } - - var idle []*autoscaler.Server - for _, server := range servers { - // skip busy servers - if _, ok := busy[server.Name]; ok { - logger.Debug(). - Str("server", server.Name). - Msg("server is busy") - continue - } - - // skip servers less than minage - if time.Now().Before(time.Unix(server.Created, 0).Add(s.Config.Pool.MinAge)) { - logger.Debug(). - Str("server", server.Name). - TimeDiff("age", time.Now(), time.Unix(server.Created, 0)). - Dur("min-age", s.Config.Pool.MinAge). - Msg("server min-age not reached") - continue - } - - idle = append(idle, server) - logger.Debug(). - Str("server", server.Name). - Msg("server is idle") - } - - // if there are no idle servers, there are no servers - // to retire and we can exit. - if len(idle) == 0 { - logger.Debug(). - Msg("no idle servers to shutdown") - return nil - } - - // we need to make sure the count of idle servers that - // are retired is > than the min pool size. - if keep := len(servers) - len(idle); keep < s.Config.Pool.Min { - idle = idle[:len(servers)-s.Config.Pool.Min-keep] - } - - var g errgroup.Group - for _, server := range idle { - var aserver = server - g.Go(func() error { - return s.collectOne(ctx, aserver) - }) - } - return g.Wait() -} - -// collectOne garbage collects a server. -func (s *Scaler) collectOne(ctx context.Context, server *autoscaler.Server) error { - logger := log.Ctx(ctx) - logger.Debug(). - Str("server", server.Name). - Msg("destroying server") - - err := s.Provider.Destroy(ctx, server) - if err != nil { - logger.Error(). - Str("server", server.Name). - Msg("failed to destroy server") - // TODO: flag the server as in an invalid state. - return err - } - logger.Debug(). - Str("server", server.Name). - Msg("destroyed server") - return s.Servers.Delete(ctx, server) -} - -// helper function returns a list of busy servers. -func (s *Scaler) listBusy(ctx context.Context) (map[string]struct{}, error) { - busy := map[string]struct{}{} - builds, err := s.Client.BuildQueue() - if err != nil { - return busy, err - } - for _, build := range builds { - if build.Status != drone.StatusRunning { - continue - } - build, err := s.Client.Build(build.Owner, build.Name, build.Number) - if err != nil { - return busy, err - } - for _, proc := range build.Procs { - busy[proc.Machine] = struct{}{} - } - } - return busy, nil -} - -// helper function returns our current capacity. -func (s *Scaler) capacity(ctx context.Context) (capacity, count int, err error) { - servers, err := s.Servers.List(ctx) - if err != nil { - return capacity, count, err - } - for _, server := range servers { - count++ - capacity += server.Capacity - } - return -} - -// helper function returns the number of pending and -// running builds in the remote Drone installation. -func (s *Scaler) count(ctx context.Context) (pending, running int, err error) { - activity, err := s.Client.BuildQueue() - if err != nil { - return pending, running, err - } - for _, activity := range activity { - if activity.Status == drone.StatusPending { - pending++ - } else { - running++ - } - } - return -} - -func requiredCapacity(pending, available, concurrency int) int { - diff := pending - available - if diff <= 0 { - return 0 - } - more := int(math.Ceil(float64(diff) / float64(concurrency))) - return more -} - -// // helper function returns true if the server is idle. -// func (s *Scaler) checkIdle(ctx context.Context, server *autoscaler.Server) bool { -// state := struct { -// Polling int `json:"polling_count"` -// Running int `json:"running_count"` -// }{} -// res, err := http.Get("http://" + server.Address + "/varz") -// if err != nil { -// return false -// } -// defer res.Body.Close() -// err = json.NewDecoder(res.Body).Decode(&state) -// if err != nil { -// return false -// } -// return state.Running > 0 -// } diff --git a/scaler/scaler_test.go b/scaler/scaler_test.go deleted file mode 100644 index bff4c677..00000000 --- a/scaler/scaler_test.go +++ /dev/null @@ -1,466 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package scaler - -import ( - "context" - "testing" - "time" - - "github.com/drone/autoscaler" - "github.com/drone/autoscaler/config" - "github.com/drone/autoscaler/mocks" - "github.com/drone/drone-go/drone" - "github.com/golang/mock/gomock" -) - -// This test verifies that if the server capacity is -// >= the pending count, and the server capacity is -// <= the pool minimum size, no actions are taken. -func TestScale_Noop(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 2}, - {Name: "server2", Capacity: 2}, - } - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return([]*drone.Activity{ - {Status: drone.StatusRunning}, - {Status: drone.StatusPending}, - {Status: drone.StatusPending}, - }, nil) - - config := config.Config{} - config.Pool.Min = 2 - scaler := Scaler{ - Client: client, - Servers: store, - Config: config, - } - - err := scaler.Scale(context.TODO()) - if err != nil { - t.Error(err) - } -} - -// This test verifies that if the server capacity is -// < than the pending count, and the server capacity is -// >= the pool maximum, no actions are taken. -func TestScale_MaxCapacity(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - // x4 capacity - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 1}, - {Name: "server2", Capacity: 1}, - {Name: "server3", Capacity: 1}, - {Name: "server4", Capacity: 1}, - } - - // x4 running builds - // x3 pending builds - builds := []*drone.Activity{ - {Status: drone.StatusRunning}, - {Status: drone.StatusRunning}, - {Status: drone.StatusRunning}, - {Status: drone.StatusRunning}, - {Status: drone.StatusPending}, - {Status: drone.StatusPending}, - {Status: drone.StatusPending}, - } - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return(builds, nil) - - config := config.Config{} - config.Pool.Min = 2 - config.Pool.Max = 4 - config.Agent.Concurrency = 2 - - scaler := Scaler{ - Client: client, - Servers: store, - Config: config, - } - - err := scaler.Scale(context.TODO()) - if err != nil { - t.Error(err) - } -} - -// This test verifies that if the server capacity is -// less than the pending count, and the server capacity is -// < the pool maximum, additional servers are provisioned. -func TestScale_MoreCapacity(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - // x2 capacity - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 1}, - {Name: "server2", Capacity: 1}, - } - - // x2 running builds - // x3 pending builds - builds := []*drone.Activity{ - {Status: drone.StatusRunning}, - {Status: drone.StatusRunning}, - {Status: drone.StatusPending}, - {Status: drone.StatusPending}, - {Status: drone.StatusPending}, - } - - // x2 mock servers provisioned - server1 := &autoscaler.Server{Name: "i-5203422c"} - server2 := &autoscaler.Server{Name: "i-4421485g"} - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - store.EXPECT().Create(gomock.Any(), server1).Return(nil) - store.EXPECT().Create(gomock.Any(), server2).Return(nil) - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return(builds, nil) - - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(server1, nil) - provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(server2, nil) - - config := config.Config{} - config.Pool.Min = 2 - config.Pool.Max = 4 - config.Agent.Concurrency = 2 - - scaler := Scaler{ - Client: client, - Servers: store, - Config: config, - Provider: provider, - } - - err := scaler.Scale(context.TODO()) - if err != nil { - t.Error(err) - } -} - -// This test verifies that if that no servers are -// destroyed if there is excess capacity and the -// the server count <= the min pool size. -func TestScale_MinPool(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - // x2 capacity - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 1}, - {Name: "server2", Capacity: 1}, - } - - // x0 running builds - // x0 pending builds - builds := []*drone.Activity{} - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return(builds, nil) - - provider := mocks.NewMockProvider(controller) - - config := config.Config{} - config.Pool.Min = 2 - config.Pool.Max = 4 - config.Agent.Concurrency = 2 - - scaler := Scaler{ - Client: client, - Servers: store, - Config: config, - Provider: provider, - } - - err := scaler.Scale(context.TODO()) - if err != nil { - t.Error(err) - } -} - -// This test verifies that if that no servers are -// destroyed if no idle servers exist. -func TestScale_NoIdle(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - // x3 capacity - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 1}, - {Name: "server2", Capacity: 1}, - {Name: "server3", Capacity: 1}, - } - - // x3 running builds - // x0 pending builds - builds := []*drone.Activity{ - {Status: drone.StatusRunning}, - {Status: drone.StatusRunning}, - {Status: drone.StatusRunning}, - } - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return(builds, nil) - client.EXPECT().BuildQueue().Return(builds, nil) - client.EXPECT().Build(gomock.Any(), gomock.Any(), gomock.Any()).Return(&drone.Build{Procs: []*drone.Proc{{Machine: "server1"}}}, nil) - client.EXPECT().Build(gomock.Any(), gomock.Any(), gomock.Any()).Return(&drone.Build{Procs: []*drone.Proc{{Machine: "server2"}}}, nil) - client.EXPECT().Build(gomock.Any(), gomock.Any(), gomock.Any()).Return(&drone.Build{Procs: []*drone.Proc{{Machine: "server3"}}}, nil) - - provider := mocks.NewMockProvider(controller) - - config := config.Config{} - config.Pool.Min = 2 - config.Pool.Max = 4 - config.Agent.Concurrency = 2 - - scaler := Scaler{ - Client: client, - Servers: store, - Config: config, - Provider: provider, - } - - err := scaler.Scale(context.TODO()) - if err != nil { - t.Error(err) - } -} - -// This test verifies that idle servers are not -// garbage collected until the min-age is reached. -func TestScale_MinAge(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - // x2 capacity - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 1, Created: time.Now().Unix()}, - {Name: "server2", Capacity: 1, Created: time.Now().Unix()}, - } - - // x0 running builds - // x0 pending builds - builds := []*drone.Activity{} - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return(builds, nil) - client.EXPECT().BuildQueue().Return(builds, nil) - - provider := mocks.NewMockProvider(controller) - - config := config.Config{} - config.Pool.Min = 1 - config.Pool.Max = 4 - config.Pool.MinAge = time.Hour - config.Agent.Concurrency = 2 - - scaler := Scaler{ - Client: client, - Servers: store, - Config: config, - Provider: provider, - } - - err := scaler.Scale(context.TODO()) - if err != nil { - t.Error(err) - } -} - -// This test verifies that idle servers are -// garbage collected while preserving the minimum -// pool size. -func TestScale_DestroyIdle(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - // x3 capacity - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 1, Created: 1}, - {Name: "server2", Capacity: 1, Created: 2}, - {Name: "server3", Capacity: 1, Created: 3}, - } - - // x0 running builds - // x0 pending builds - builds := []*drone.Activity{} - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - store.EXPECT().Delete(gomock.Any(), servers[1]).Return(nil) - store.EXPECT().Delete(gomock.Any(), servers[2]).Return(nil) - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return(builds, nil) - client.EXPECT().BuildQueue().Return(builds, nil) - - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Destroy(gomock.Any(), servers[1]).Return(nil) - provider.EXPECT().Destroy(gomock.Any(), servers[2]).Return(nil) - - config := config.Config{} - config.Pool.Min = 1 - config.Pool.Max = 4 - config.Agent.Concurrency = 2 - - scaler := Scaler{ - Client: client, - Servers: store, - Config: config, - Provider: provider, - } - - err := scaler.Scale(context.TODO()) - if err != nil { - t.Error(err) - } -} - -func TestListBusy(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - client := mocks.NewMockClient(controller) - client.EXPECT().Build("octocat", "hello-world", 1).Return(&drone.Build{ - Procs: []*drone.Proc{ - {PID: 1, Machine: "machine1"}, - {PID: 2, Machine: "machine2"}, - }, - }, nil) - client.EXPECT().BuildQueue().Return([]*drone.Activity{ - {Status: drone.StatusPending}, - {Status: drone.StatusRunning, Owner: "octocat", Name: "hello-world", Number: 1}, - }, nil) - - scaler := Scaler{Client: client} - busy, err := scaler.listBusy(context.TODO()) - if err != nil { - t.Error(err) - return - } - if got, want := len(busy), 2; got != want { - t.Errorf("Want busy server count %d, got %d", want, got) - } - if _, ok := busy["machine1"]; !ok { - t.Errorf("Expected server not in busy list") - } - if _, ok := busy["machine2"]; !ok { - t.Errorf("Expected server not in busy list") - } -} - -func TestCapacity(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - servers := []*autoscaler.Server{ - {Name: "server1", Capacity: 4}, - {Name: "server2", Capacity: 3}, - {Name: "server3", Capacity: 2}, - {Name: "server4", Capacity: 1}, - } - - store := mocks.NewMockServerStore(controller) - store.EXPECT().List(gomock.Any()).Return(servers, nil) - - scaler := Scaler{Servers: store} - capacity, count, err := scaler.capacity(context.TODO()) - if err != nil { - t.Error(err) - return - } - if got, want := capacity, 10; got != want { - t.Errorf("Want capacity count %d, got %d", want, got) - } - if got, want := count, 4; got != want { - t.Errorf("Want server count %d, got %d", want, got) - } -} - -func TestCount(t *testing.T) { - controller := gomock.NewController(t) - defer controller.Finish() - - client := mocks.NewMockClient(controller) - client.EXPECT().BuildQueue().Return([]*drone.Activity{ - {Status: drone.StatusPending}, - {Status: drone.StatusPending}, - {Status: drone.StatusPending}, - {Status: drone.StatusRunning}, - {Status: drone.StatusRunning}, - }, nil) - - scaler := Scaler{Client: client} - pending, running, err := scaler.count(context.TODO()) - if err != nil { - t.Error(err) - return - } - if got, want := pending, 3; got != want { - t.Errorf("Want pending count %d, got %d", want, got) - } - if got, want := running, 2; got != want { - t.Errorf("Want running count %d, got %d", want, got) - } -} - -func TestRequiredCapacity(t *testing.T) { - tests := []struct { - pending, - available, - concurrency, - want int - }{ - {0, 2, 2, 0}, // no pending builds - {2, 2, 2, 0}, // use 2 of 2 existing - {1, 2, 2, 0}, // use 1 of 2 existing - {4, 2, 2, 1}, // want 2 servers - {4, 2, 1, 2}, // want 2 servers - {5, 2, 2, 2}, // want 2 servers (round-up) - } - for _, test := range tests { - capacity := requiredCapacity( - test.pending, - test.available, - test.concurrency, - ) - if got, want := capacity, test.want; got != want { - t.Errorf("Got capacity %d, want %d", got, want) - } - } -} diff --git a/server.go b/server.go index 906ee5de..a6d710f5 100644 --- a/server.go +++ b/server.go @@ -7,8 +7,20 @@ package autoscaler import ( "context" "errors" +) + +// ServerState specifies the server state. +type ServerState string - "github.com/dchest/uniuri" +// ServerState type enumeration. +const ( + StatePending = ServerState("pending") + StateStaging = ServerState("staging") + StateRunning = ServerState("running") + StateShutdown = ServerState("shutdown") + StateStopping = ServerState("stopping") + StateStopped = ServerState("stopped") + StateError = ServerState("error") ) // ErrServerNotFound is returned when the requested server @@ -20,9 +32,12 @@ type ServerStore interface { // Find a server by unique name. Find(context.Context, string) (*Server, error) - // List all registered servers + // List returns all registered servers List(context.Context) ([]*Server, error) + // ListState returns all servers with the given state. + ListState(context.Context, ServerState) ([]*Server, error) + // Create the server record in the store. Create(context.Context, *Server) error @@ -31,48 +46,26 @@ type ServerStore interface { // Delete the server record from the store. Delete(context.Context, *Server) error + + // Purge old server records from the store. + Purge(context.Context, int64) error } // Server stores the server details. type Server struct { - Provider ProviderType `json:"provider"` - UID string `json:"uid"` - Name string `json:"name"` - Image string `json:"image"` - Region string `json:"region"` - Size string `json:"size"` - Address string `json:"address"` - Secret string `json:"secret"` - Capacity int `json:"capacity"` - Active bool `json:"active"` - Healthy bool `json:"healthy"` - Created int64 `json:"created"` - Updated int64 `json:"updated"` - Logs string `json:"-"` -} - -// ByCreated sorts the server list by created date. -type ByCreated []*Server - -func (a ByCreated) Len() int { return len(a) } -func (a ByCreated) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ByCreated) Less(i, j int) bool { return a[i].Created < a[j].Created } - -// ServerOpts defines server creation options. -type ServerOpts struct { - Name string - Secret string - Capacity int -} - -// NewServerOpts returns server options with a unique -// server identifier and designated capacity. -func NewServerOpts(prefix string, capacity int) *ServerOpts { - suffix := uniuri.NewLen(5) - secret := uniuri.New() - return &ServerOpts{ - Name: prefix + "-" + suffix, - Secret: secret, - Capacity: capacity, - } + ID string `db:"server_id" json:"id"` + Provider ProviderType `db:"server_provider" json:"provider"` + State ServerState `db:"server_state" json:"state"` + Name string `db:"server_name" json:"name"` + Image string `db:"server_image" json:"image"` + Region string `db:"server_region" json:"region"` + Size string `db:"server_size" json:"size"` + Address string `db:"server_address" json:"address"` + Capacity int `db:"server_capacity" json:"capacity"` + Secret string `db:"server_secret" json:"secret"` + Error string `db:"server_error" json:"Error"` + Created int64 `db:"server_created" json:"created"` + Updated int64 `db:"server_updated" json:"updated"` + Started int64 `db:"server_started" json:"started"` + Stopped int64 `db:"server_stopped" json:"stopped"` } diff --git a/server/scaler.go b/server/engine.go similarity index 53% rename from server/scaler.go rename to server/engine.go index 3e02580e..858bb511 100644 --- a/server/scaler.go +++ b/server/engine.go @@ -10,20 +10,20 @@ import ( "github.com/drone/autoscaler" ) -// HandleScalerPause returns an http.HandlerFunc that pauses -// automatic scaling. -func HandleScalerPause(scaler autoscaler.Scaler) http.HandlerFunc { +// HandleEnginePause returns an http.HandlerFunc that pauses +// scaling engine. +func HandleEnginePause(engine autoscaler.Engine) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - scaler.Pause() + engine.Pause() w.WriteHeader(204) } } -// HandleScalerResume returns an http.HandlerFunc that resumed -// automatic scaling. -func HandleScalerResume(scaler autoscaler.Scaler) http.HandlerFunc { +// HandleEngineResume returns an http.HandlerFunc that resumes +// scaling engine. +func HandleEngineResume(engine autoscaler.Engine) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - scaler.Resume() + engine.Resume() w.WriteHeader(204) } } diff --git a/server/scaler_test.go b/server/engine_test.go similarity index 63% rename from server/scaler_test.go rename to server/engine_test.go index f2e8c1d1..cdcd1200 100644 --- a/server/scaler_test.go +++ b/server/engine_test.go @@ -9,42 +9,37 @@ import ( "testing" "github.com/drone/autoscaler/mocks" - "github.com/go-chi/chi" "github.com/golang/mock/gomock" ) -func TestHandleScalerPause(t *testing.T) { +func TestHandleEnginePause(t *testing.T) { controller := gomock.NewController(t) defer controller.Finish() w := httptest.NewRecorder() r := httptest.NewRequest("POST", "/api/pause", nil) - scaler := mocks.NewMockScaler(controller) - scaler.EXPECT().Pause() + e := mocks.NewMockEngine(controller) + e.EXPECT().Pause() - router := chi.NewRouter() - router.Post("/api/pause", HandleScalerPause(scaler)) - router.ServeHTTP(w, r) + HandleEnginePause(e).ServeHTTP(w, r) if got, want := w.Code, 204; want != got { t.Errorf("Want response code %d, got %d", want, got) } } -func TestHandleScalerResume(t *testing.T) { +func TestHandleEngineResume(t *testing.T) { controller := gomock.NewController(t) defer controller.Finish() w := httptest.NewRecorder() r := httptest.NewRequest("POST", "/api/resume", nil) - scaler := mocks.NewMockScaler(controller) - scaler.EXPECT().Resume() + e := mocks.NewMockEngine(controller) + e.EXPECT().Resume() - router := chi.NewRouter() - router.Post("/api/resume", HandleScalerResume(scaler)) - router.ServeHTTP(w, r) + HandleEngineResume(e).ServeHTTP(w, r) if got, want := w.Code, 204; want != got { t.Errorf("Want response code %d, got %d", want, got) diff --git a/server/healthz_test.go b/server/healthz_test.go index 1a2ed9a0..89dae15c 100644 --- a/server/healthz_test.go +++ b/server/healthz_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "testing" - "github.com/go-chi/chi" "github.com/golang/mock/gomock" ) @@ -19,9 +18,7 @@ func TestHandleHealthz(t *testing.T) { w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/healthz", nil) - router := chi.NewRouter() - router.Get("/healthz", HandleHealthz()) - router.ServeHTTP(w, r) + HandleHealthz().ServeHTTP(w, r) if got, want := w.Code, 200; want != got { t.Errorf("Want response code %d, got %d", want, got) diff --git a/server/queue_test.go b/server/queue_test.go index 76e8f5f7..a656e2a3 100644 --- a/server/queue_test.go +++ b/server/queue_test.go @@ -13,7 +13,6 @@ import ( "github.com/drone/autoscaler/mocks" "github.com/drone/drone-go/drone" - "github.com/go-chi/chi" "github.com/golang/mock/gomock" "github.com/kr/pretty" ) @@ -40,9 +39,7 @@ func TestHandleQueueList(t *testing.T) { client := mocks.NewMockClient(controller) client.EXPECT().BuildQueue().Return(mockBuilds, nil) - router := chi.NewRouter() - router.Get("/api/queue", HandleQueueList(client)) - router.ServeHTTP(w, r) + HandleQueueList(client).ServeHTTP(w, r) if got, want := w.Code, 200; want != got { t.Errorf("Want response code %d, got %d", want, got) @@ -67,9 +64,7 @@ func TestHandleQueueListErr(t *testing.T) { client := mocks.NewMockClient(controller) client.EXPECT().BuildQueue().Return(nil, err) - router := chi.NewRouter() - router.Get("/api/queue", HandleQueueList(client)) - router.ServeHTTP(w, r) + HandleQueueList(client).ServeHTTP(w, r) if got, want := w.Code, 500; want != got { t.Errorf("Want response code %d, got %d", want, got) diff --git a/server/servers.go b/server/servers.go index 486a375f..4d564970 100644 --- a/server/servers.go +++ b/server/servers.go @@ -7,6 +7,7 @@ package server import ( "net/http" + "github.com/dchest/uniuri" "github.com/drone/autoscaler" "github.com/drone/autoscaler/config" @@ -56,7 +57,6 @@ func HandleServerFind(servers autoscaler.ServerStore) http.HandlerFunc { // and then deletes the named server. func HandleServerDelete( servers autoscaler.ServerStore, - provider autoscaler.Provider, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -71,27 +71,18 @@ func HandleServerDelete( writeNotFound(w, err) return } - err = provider.Destroy(ctx, server) + server.State = autoscaler.StateShutdown + err = servers.Update(ctx, server) if err != nil { hlog.FromRequest(r). Error(). Err(err). Str("server", name). - Msg("cannot kill server") + Msg("cannot update server") writeError(w, err) return } - err = servers.Delete(ctx, server) - if err != nil { - hlog.FromRequest(r). - Error(). - Err(err). - Str("server", name). - Msg("cannot purge server from datastore") - writeError(w, err) - return - } - w.WriteHeader(204) + writeJSON(w, server, 200) } } @@ -99,22 +90,16 @@ func HandleServerDelete( // and a new server. func HandleServerCreate( servers autoscaler.ServerStore, - provider autoscaler.Provider, config config.Config, ) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - opt := autoscaler.NewServerOpts("agent", config.Agent.Concurrency) - server, err := provider.Create(ctx, opt) - if err != nil { - hlog.FromRequest(r). - Error(). - Err(err). - Msg("cannot create server") - writeError(w, err) - return + server := &autoscaler.Server{ + Name: "agent-" + uniuri.NewLen(8), + State: autoscaler.StatePending, + Capacity: config.Agent.Concurrency, } - err = servers.Create(ctx, server) + err := servers.Create(ctx, server) if err != nil { hlog.FromRequest(r). Error(). diff --git a/server/servers_test.go b/server/servers_test.go index 4524fd1f..486729bb 100644 --- a/server/servers_test.go +++ b/server/servers_test.go @@ -34,9 +34,7 @@ func TestHandleServerList(t *testing.T) { store := mocks.NewMockServerStore(controller) store.EXPECT().List(gomock.Any()).Return(servers, nil) - router := chi.NewRouter() - router.Get("/api/servers", HandleServerList(store)) - router.ServeHTTP(w, r) + HandleServerList(store).ServeHTTP(w, r) if got, want := w.Code, 200; want != got { t.Errorf("Want response code %d, got %d", want, got) @@ -61,9 +59,7 @@ func TestHandleServerListErr(t *testing.T) { store := mocks.NewMockServerStore(controller) store.EXPECT().List(gomock.Any()).Return(nil, err) - router := chi.NewRouter() - router.Get("/api/servers", HandleServerList(store)) - router.ServeHTTP(w, r) + HandleServerList(store).ServeHTTP(w, r) if got, want := w.Code, 500; want != got { t.Errorf("Want response code %d, got %d", want, got) @@ -136,33 +132,14 @@ func TestHandleServerCreate(t *testing.T) { w := httptest.NewRecorder() r := httptest.NewRequest("POST", "/api/servers", nil) - server := &autoscaler.Server{ - Name: "i-5203422c", - Image: "docker-16-04", - Region: "nyc1", - Size: "s-1vcpu-1gb", - } - - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(server, nil) - store := mocks.NewMockServerStore(controller) - store.EXPECT().Create(gomock.Any(), server).Return(nil) + store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil) - router := chi.NewRouter() - router.Post("/api/servers", HandleServerCreate(store, provider, config.Config{})) - router.ServeHTTP(w, r) + HandleServerCreate(store, config.Config{}).ServeHTTP(w, r) if got, want := w.Code, 200; want != got { t.Errorf("Want response code %d, got %d", want, got) } - - got, want := &autoscaler.Server{}, server - json.NewDecoder(w.Body).Decode(got) - if !reflect.DeepEqual(got, want) { - t.Errorf("response body does match expected result") - pretty.Ldiff(t, got, want) - } } func TestHandleServerCreateFailure(t *testing.T) { @@ -173,14 +150,11 @@ func TestHandleServerCreateFailure(t *testing.T) { r := httptest.NewRequest("POST", "/api/servers", nil) err := errors.New("oops") - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil, err) - store := mocks.NewMockServerStore(controller) + store.EXPECT().Create(gomock.Any(), gomock.Any()).Return(err) - router := chi.NewRouter() - router.Post("/api/servers", HandleServerCreate(store, provider, config.Config{})) - router.ServeHTTP(w, r) + h := HandleServerCreate(store, config.Config{}) + h.ServeHTTP(w, r) if got, want := w.Code, 500; want != got { t.Errorf("Want response code %d, got %d", want, got) @@ -207,20 +181,20 @@ func TestHandleServerDelete(t *testing.T) { Size: "s-1vcpu-1gb", } - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Destroy(gomock.Any(), server).Return(nil) - store := mocks.NewMockServerStore(controller) store.EXPECT().Find(gomock.Any(), server.Name).Return(server, nil) - store.EXPECT().Delete(gomock.Any(), server).Return(nil) + store.EXPECT().Update(gomock.Any(), server).Return(nil) router := chi.NewRouter() - router.Delete("/api/servers/{name}", HandleServerDelete(store, provider)) + router.Delete("/api/servers/{name}", HandleServerDelete(store)) router.ServeHTTP(w, r) - if got, want := w.Code, 204; want != got { + if got, want := w.Code, 200; want != got { t.Errorf("Want response code %d, got %d", want, got) } + if got, want := server.State, autoscaler.StateShutdown; got != want { + t.Errorf("Want server state Shutdown, got %d", got) + } } func TestHandleServerDeleteNotFound(t *testing.T) { @@ -231,13 +205,12 @@ func TestHandleServerDeleteNotFound(t *testing.T) { r := httptest.NewRequest("DELETE", "/api/servers/i-5203422c", nil) err := errors.New("not found") - provider := mocks.NewMockProvider(controller) store := mocks.NewMockServerStore(controller) store.EXPECT().Find(gomock.Any(), "i-5203422c").Return(nil, err) router := chi.NewRouter() - router.Delete("/api/servers/{name}", HandleServerDelete(store, provider)) + router.Delete("/api/servers/{name}", HandleServerDelete(store)) router.ServeHTTP(w, r) if got, want := w.Code, 404; want != got { @@ -266,14 +239,13 @@ func TestHandleServerDeleteFailure(t *testing.T) { } err := errors.New("bad request") - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Destroy(gomock.Any(), server).Return(err) store := mocks.NewMockServerStore(controller) store.EXPECT().Find(gomock.Any(), server.Name).Return(server, nil) + store.EXPECT().Update(gomock.Any(), server).Return(err) router := chi.NewRouter() - router.Delete("/api/servers/{name}", HandleServerDelete(store, provider)) + router.Delete("/api/servers/{name}", HandleServerDelete(store)) router.ServeHTTP(w, r) if got, want := w.Code, 500; want != got { diff --git a/server/varz.go b/server/varz.go index 62c97ecd..c184a2e5 100644 --- a/server/varz.go +++ b/server/varz.go @@ -16,10 +16,10 @@ type varz struct { // HandleVarz creates an http.HandlerFunc that returns system // configuration and runtime information. -func HandleVarz(scaler autoscaler.Scaler) http.HandlerFunc { +func HandleVarz(engine autoscaler.Engine) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { data := varz{ - Paused: scaler.Paused(), + Paused: engine.Paused(), } writeJSON(w, &data, 200) } diff --git a/server/varz_test.go b/server/varz_test.go index 01c51cb6..298f537d 100644 --- a/server/varz_test.go +++ b/server/varz_test.go @@ -27,11 +27,11 @@ func TestHandleVarz(t *testing.T) { w := httptest.NewRecorder() r := httptest.NewRequest("POST", "/varz", nil) - scaler := mocks.NewMockScaler(controller) - scaler.EXPECT().Paused().Return(true) + engine := mocks.NewMockEngine(controller) + engine.EXPECT().Paused().Return(true) router := chi.NewRouter() - router.Post("/varz", HandleVarz(scaler)) + router.Post("/varz", HandleVarz(engine)) router.ServeHTTP(w, r) if got, want := w.Code, 200; want != got { diff --git a/server/version_test.go b/server/version_test.go index 88ab9b9d..c6b2482e 100644 --- a/server/version_test.go +++ b/server/version_test.go @@ -10,7 +10,6 @@ import ( "reflect" "testing" - "github.com/go-chi/chi" "github.com/golang/mock/gomock" "github.com/kr/pretty" ) @@ -28,9 +27,8 @@ func TestHandleVersion(t *testing.T) { Commit: "ad2aec", } - router := chi.NewRouter() - router.Get("/version", HandleVersion(mockVersion.Source, mockVersion.Version, mockVersion.Commit)) - router.ServeHTTP(w, r) + h := HandleVersion(mockVersion.Source, mockVersion.Version, mockVersion.Commit) + h.ServeHTTP(w, r) if got, want := w.Code, 200; want != got { t.Errorf("Want response code %d, got %d", want, got) diff --git a/server_test.go b/server_test.go deleted file mode 100644 index 2adf5fde..00000000 --- a/server_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package autoscaler - -import ( - "sort" - "strings" - "testing" -) - -func TestByCreated(t *testing.T) { - servers := []*Server{ - {Created: 4, Name: "fourth"}, - {Created: 2, Name: "second"}, - {Created: 3, Name: "third"}, - {Created: 5, Name: "fifth"}, - {Created: 1, Name: "first"}, - } - - sort.Sort(ByCreated(servers)) - - for i, server := range servers { - if server.Created != int64(i+1) { - t.Errorf("Invalid sort order %d for %q", i, server.Name) - } - } -} - -func TestServerOpts(t *testing.T) { - opts := NewServerOpts("agent", 4) - if got, want := opts.Capacity, 4; got != want { - t.Errorf("Want capacity %d, got %d", want, got) - } - if !strings.HasPrefix(opts.Name, "agent-") { - t.Errorf("Want server name prefixed with %s, got %s", "agent-", opts.Name) - } - if got, want := len(opts.Name), len("agent-")+5; got != want { - t.Errorf("Want server name length %d, got %d", want, got) - } -} diff --git a/slack/slack.go b/slack/slack.go index 142dac8c..f2883400 100644 --- a/slack/slack.go +++ b/slack/slack.go @@ -20,30 +20,25 @@ import ( // New returns a new provider that is instrumented to send // Slack notifications when server instances are provisioned // or terminated. -func New(config config.Config, base autoscaler.Provider) autoscaler.Provider { +func New(config config.Config, base autoscaler.ServerStore) autoscaler.ServerStore { return ¬ifier{ - Provider: base, - client: slack.NewWebHook(config.Slack.Webhook), + ServerStore: base, + client: slack.NewWebHook(config.Slack.Webhook), } } type notifier struct { - autoscaler.Provider + autoscaler.ServerStore client *slack.WebHook channel string } -func (n *notifier) Create(ctx context.Context, opts *autoscaler.ServerOpts) (*autoscaler.Server, error) { - server, err := n.Provider.Create(ctx, opts) - if err == nil { +func (n *notifier) Update(ctx context.Context, server *autoscaler.Server) error { + err := n.ServerStore.Update(ctx, server) + switch { + case server.State == autoscaler.StateRunning: n.notifyCreate(server) - } - return server, err -} - -func (n *notifier) Destroy(ctx context.Context, server *autoscaler.Server) error { - err := n.Provider.Destroy(ctx, server) - if err == nil { + case server.State == autoscaler.StateStopped: n.notifyDestroy(server) } return err diff --git a/slack/slack_test.go b/slack/slack_test.go index 36d7041d..c7380c5e 100644 --- a/slack/slack_test.go +++ b/slack/slack_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/h2non/gock" - "github.com/kr/pretty" "github.com/drone/autoscaler" "github.com/drone/autoscaler/config" @@ -30,7 +29,7 @@ func TestHumanizeTime(t *testing.T) { } } -func TestCreate(t *testing.T) { +func TestUpdateRunning(t *testing.T) { defer gock.Off() controller := gomock.NewController(t) @@ -40,11 +39,7 @@ func TestCreate(t *testing.T) { Name: "this-is-a-test-message", Region: "nyc1", Size: "s-1vcpu-1gb", - } - - opts := &autoscaler.ServerOpts{ - Name: "i-123789331", - Capacity: 2, + State: autoscaler.StateRunning, } // TODO: verify the contents of the Slack payload. @@ -56,21 +51,17 @@ func TestCreate(t *testing.T) { conf := config.Config{} conf.Slack.Webhook = "https://hooks.slack.com/services/XXX/YYY/ZZZ" - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(server, nil) + store := mocks.NewMockServerStore(controller) + store.EXPECT().Update(gomock.Any(), server).Return(nil) - slack := New(conf, provider) - result, err := slack.Create(noContext, opts) + slack := New(conf, store) + err := slack.Update(noContext, server) if err != nil { t.Error(err) } - if got, want := result, server; got != want { - t.Errorf("Unexpected response") - pretty.Ldiff(t, got, want) - } } -func TestDestroy(t *testing.T) { +func TestUpdateStopped(t *testing.T) { defer gock.Off() controller := gomock.NewController(t) @@ -80,6 +71,7 @@ func TestDestroy(t *testing.T) { Name: "this-is-a-test-message", Region: "nyc1", Size: "s-1vcpu-1gb", + State: autoscaler.StateStopped, } // TODO: verify the contents of the Slack payload. @@ -91,11 +83,11 @@ func TestDestroy(t *testing.T) { conf := config.Config{} conf.Slack.Webhook = "https://hooks.slack.com/services/XXX/YYY/ZZZ" - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Destroy(gomock.Any(), server).Return(nil) + store := mocks.NewMockServerStore(controller) + store.EXPECT().Update(gomock.Any(), server).Return(nil) - slack := New(conf, provider) - err := slack.Destroy(noContext, server) + slack := New(conf, store) + err := slack.Update(noContext, server) if err != nil { t.Error(err) } @@ -115,29 +107,23 @@ func TestIntegration(t *testing.T) { defer controller.Finish() server := &autoscaler.Server{ - Name: "this-is-a-test-message", - Region: "nyc1", - Size: "s-1vcpu-1gb", - } - - opts := &autoscaler.ServerOpts{ Name: "i-123789331", + Address: "1.2.3.4", + Region: "nyc1", + Size: "s-1vcpu-1gb", Capacity: 2, + State: autoscaler.StateRunning, } conf := config.Config{} conf.Slack.Webhook = webhook - provider := mocks.NewMockProvider(controller) - provider.EXPECT().Create(gomock.Any(), gomock.Any()).Return(server, nil) + store := mocks.NewMockServerStore(controller) + store.EXPECT().Update(gomock.Any(), server).Return(nil) - slack := New(conf, provider) - result, err := slack.Create(noContext, opts) + slack := New(conf, store) + err := slack.Update(noContext, server) if err != nil { t.Error(err) } - - if result != server { - t.Errorf("Invalid response") - } } diff --git a/store/db.go b/store/db.go new file mode 100644 index 00000000..46e37a1f --- /dev/null +++ b/store/db.go @@ -0,0 +1,65 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package store + +import ( + "database/sql" + "time" + + "github.com/drone/autoscaler/store/migrate" + + "github.com/jmoiron/sqlx" +) + +// Connect to a database and verify with a ping. +func Connect(driver, datasource string) (*sqlx.DB, error) { + db, err := sql.Open(driver, datasource) + if err != nil { + return nil, err + } + switch driver { + case "mysql": + db.SetMaxIdleConns(0) + case "sqlite3": + db.SetMaxOpenConns(1) + } + dbx := sqlx.NewDb(db, driver) + if err := pingDatabase(dbx); err != nil { + return nil, err + } + if err := setupDatabase(dbx); err != nil { + return nil, err + } + return dbx, nil +} + +// Must is a helper function that wraps a call to Connect +// and panics if the error is non-nil. +func Must(db *sqlx.DB, err error) *sqlx.DB { + if err != nil { + panic(err) + } + return db +} + +// helper function to ping the database with backoff to ensure +// a connection can be established before we proceed with the +// database setup and migration. +func pingDatabase(db *sqlx.DB) (err error) { + for i := 0; i < 30; i++ { + err = db.Ping() + if err == nil { + return + } + time.Sleep(time.Second) + } + return +} + +// helper function to setup the databsae by performing automated +// database migration steps. +func setupDatabase(db *sqlx.DB) error { + return ddl.Migrate(db) +} diff --git a/store/db_test.go b/store/db_test.go new file mode 100644 index 00000000..2f18ce88 --- /dev/null +++ b/store/db_test.go @@ -0,0 +1,30 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package store + +import ( + "context" + "os" + + "github.com/jmoiron/sqlx" + + _ "github.com/go-sql-driver/mysql" + _ "github.com/mattn/go-sqlite3" +) + +var noContext = context.Background() + +// connect opens a new test database connection. +func connect() (*sqlx.DB, error) { + var ( + driver = "sqlite3" + config = ":memory:" + ) + if os.Getenv("DATABASE_DRIVER") != "" { + driver = os.Getenv("DATABASE_DRIVER") + config = os.Getenv("DATABASE_CONFIG") + } + return Connect(driver, config) +} diff --git a/store/migrate/migrate.go b/store/migrate/migrate.go new file mode 100644 index 00000000..3ce88055 --- /dev/null +++ b/store/migrate/migrate.go @@ -0,0 +1,18 @@ +package ddl + +import ( + "github.com/drone/autoscaler/store/migrate/mysql" + "github.com/drone/autoscaler/store/migrate/sqlite" + + "github.com/jmoiron/sqlx" +) + +// Migrate performs the database migration. +func Migrate(db *sqlx.DB) error { + switch db.DriverName() { + case "mysql": + return mysql.Migrate(db.DB) + default: + return sqlite.Migrate(db.DB) + } +} diff --git a/store/migrate/mysql/ddl.go b/store/migrate/mysql/ddl.go new file mode 100644 index 00000000..5c0df902 --- /dev/null +++ b/store/migrate/mysql/ddl.go @@ -0,0 +1,7 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package mysql + +//go:generate togo ddl -package mysql -dialect mysql diff --git a/store/migrate/mysql/ddl_gen.go b/store/migrate/mysql/ddl_gen.go new file mode 100644 index 00000000..1a9753ac --- /dev/null +++ b/store/migrate/mysql/ddl_gen.go @@ -0,0 +1,128 @@ +package mysql + +import ( + "database/sql" +) + +var migrations = []struct { + name string + stmt string +}{ + { + name: "create-table-servers", + stmt: createTableServers, + }, + { + name: "create-index-server-id", + stmt: createIndexServerId, + }, + { + name: "create-index-server-state", + stmt: createIndexServerState, + }, +} + +// Migrate performs the database migration. If the migration fails +// and error is returned. +func Migrate(db *sql.DB) error { + if err := createTable(db); err != nil { + return err + } + completed, err := selectCompleted(db) + if err != nil && err != sql.ErrNoRows { + return err + } + for _, migration := range migrations { + if _, ok := completed[migration.name]; ok { + + continue + } + + if _, err := db.Exec(migration.stmt); err != nil { + return err + } + if err := insertMigration(db, migration.name); err != nil { + return err + } + + } + return nil +} + +func createTable(db *sql.DB) error { + _, err := db.Exec(migrationTableCreate) + return err +} + +func insertMigration(db *sql.DB, name string) error { + _, err := db.Exec(migrationInsert, name) + return err +} + +func selectCompleted(db *sql.DB) (map[string]struct{}, error) { + migrations := map[string]struct{}{} + rows, err := db.Query(migrationSelect) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + migrations[name] = struct{}{} + } + return migrations, nil +} + +// +// migration table ddl and sql +// + +var migrationTableCreate = ` +CREATE TABLE IF NOT EXISTS migrations ( + name VARCHAR(255) +,UNIQUE(name) +) +` + +var migrationInsert = ` +INSERT INTO migrations (name) VALUES (?) +` + +var migrationSelect = ` +SELECT name FROM migrations +` + +// +// 001_create_table_servers.sql +// + +var createTableServers = ` +CREATE TABLE IF NOT EXISTS servers ( + server_name VARCHAR(50) PRIMARY KEY +,server_id VARCHAR(250) +,server_provider VARCHAR(50) +,server_state VARCHAR(50) +,server_image VARCHAR(250) +,server_region VARCHAR(50) +,server_size VARCHAR(50) +,server_address VARCHAR(250) +,server_capacity INTEGER +,server_secret VARCHAR(50) +,server_error MEDIUMBLOB +,server_created INTEGER +,server_updated INTEGER +,server_started INTEGER +,server_stopped INTEGER +); +` + +var createIndexServerId = ` +CREATE INDEX IF NOT EXISTS ix_servers_id ON servers (server_id); +` + +var createIndexServerState = ` +CREATE INDEX IF NOT EXISTS ix_servers_state ON servers (server_state); +` diff --git a/store/migrate/mysql/files/001_create_table_servers.sql b/store/migrate/mysql/files/001_create_table_servers.sql new file mode 100644 index 00000000..0a73826a --- /dev/null +++ b/store/migrate/mysql/files/001_create_table_servers.sql @@ -0,0 +1,27 @@ +-- name: create-table-servers + +CREATE TABLE IF NOT EXISTS servers ( + server_name VARCHAR(50) PRIMARY KEY +,server_id VARCHAR(250) +,server_provider VARCHAR(50) +,server_state VARCHAR(50) +,server_image VARCHAR(250) +,server_region VARCHAR(50) +,server_size VARCHAR(50) +,server_address VARCHAR(250) +,server_capacity INTEGER +,server_secret VARCHAR(50) +,server_error MEDIUMBLOB +,server_created INTEGER +,server_updated INTEGER +,server_started INTEGER +,server_stopped INTEGER +); + +-- name: create-index-server-id + +CREATE INDEX IF NOT EXISTS ix_servers_id ON servers (server_id); + +-- name: create-index-server-state + +CREATE INDEX IF NOT EXISTS ix_servers_state ON servers (server_state); diff --git a/store/migrate/sqlite/ddl.go b/store/migrate/sqlite/ddl.go new file mode 100644 index 00000000..fa82dd1a --- /dev/null +++ b/store/migrate/sqlite/ddl.go @@ -0,0 +1,7 @@ +// Copyright 2018 Drone.IO Inc +// Use of this software is governed by the Business Source License +// that can be found in the LICENSE file. + +package sqlite + +//go:generate togo ddl -package sqlite -dialect sqlite3 diff --git a/store/migrate/sqlite/ddl_gen.go b/store/migrate/sqlite/ddl_gen.go new file mode 100644 index 00000000..56b50ea9 --- /dev/null +++ b/store/migrate/sqlite/ddl_gen.go @@ -0,0 +1,128 @@ +package sqlite + +import ( + "database/sql" +) + +var migrations = []struct { + name string + stmt string +}{ + { + name: "create-table-servers", + stmt: createTableServers, + }, + { + name: "create-index-server-id", + stmt: createIndexServerId, + }, + { + name: "create-index-server-state", + stmt: createIndexServerState, + }, +} + +// Migrate performs the database migration. If the migration fails +// and error is returned. +func Migrate(db *sql.DB) error { + if err := createTable(db); err != nil { + return err + } + completed, err := selectCompleted(db) + if err != nil && err != sql.ErrNoRows { + return err + } + for _, migration := range migrations { + if _, ok := completed[migration.name]; ok { + + continue + } + + if _, err := db.Exec(migration.stmt); err != nil { + return err + } + if err := insertMigration(db, migration.name); err != nil { + return err + } + + } + return nil +} + +func createTable(db *sql.DB) error { + _, err := db.Exec(migrationTableCreate) + return err +} + +func insertMigration(db *sql.DB, name string) error { + _, err := db.Exec(migrationInsert, name) + return err +} + +func selectCompleted(db *sql.DB) (map[string]struct{}, error) { + migrations := map[string]struct{}{} + rows, err := db.Query(migrationSelect) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, err + } + migrations[name] = struct{}{} + } + return migrations, nil +} + +// +// migration table ddl and sql +// + +var migrationTableCreate = ` +CREATE TABLE IF NOT EXISTS migrations ( + name VARCHAR(255) +,UNIQUE(name) +) +` + +var migrationInsert = ` +INSERT INTO migrations (name) VALUES (?) +` + +var migrationSelect = ` +SELECT name FROM migrations +` + +// +// 001_create_table_servers.sql +// + +var createTableServers = ` +CREATE TABLE IF NOT EXISTS servers ( + server_name TEXT PRIMARY KEY +,server_id TEXT +,server_provider TEXT +,server_state TEXT +,server_image TEXT +,server_region TEXT +,server_size TEXT +,server_address TEXT +,server_capacity INTEGER +,server_secret TEXT +,server_error TEXT +,server_created INTEGER +,server_updated INTEGER +,server_started INTEGER +,server_stopped INTEGER +); +` + +var createIndexServerId = ` +CREATE INDEX IF NOT EXISTS ix_servers_id ON servers (server_id); +` + +var createIndexServerState = ` +CREATE INDEX IF NOT EXISTS ix_servers_state ON servers (server_state); +` diff --git a/store/migrate/sqlite/files/001_create_table_servers.sql b/store/migrate/sqlite/files/001_create_table_servers.sql new file mode 100644 index 00000000..68e147b9 --- /dev/null +++ b/store/migrate/sqlite/files/001_create_table_servers.sql @@ -0,0 +1,27 @@ +-- name: create-table-servers + +CREATE TABLE IF NOT EXISTS servers ( + server_name TEXT PRIMARY KEY +,server_id TEXT +,server_provider TEXT +,server_state TEXT +,server_image TEXT +,server_region TEXT +,server_size TEXT +,server_address TEXT +,server_capacity INTEGER +,server_secret TEXT +,server_error TEXT +,server_created INTEGER +,server_updated INTEGER +,server_started INTEGER +,server_stopped INTEGER +); + +-- name: create-index-server-id + +CREATE INDEX IF NOT EXISTS ix_servers_id ON servers (server_id); + +-- name: create-index-server-state + +CREATE INDEX IF NOT EXISTS ix_servers_state ON servers (server_state); diff --git a/store/servers.go b/store/servers.go index d1f06a46..c5198202 100644 --- a/store/servers.go +++ b/store/servers.go @@ -6,63 +6,216 @@ package store import ( "context" - "encoding/json" + "database/sql" + "time" - "github.com/boltdb/bolt" "github.com/drone/autoscaler" + + "github.com/jmoiron/sqlx" ) // NewServerStore returns a new server store. -func NewServerStore(db *bolt.DB) autoscaler.ServerStore { +func NewServerStore(db *sqlx.DB) autoscaler.ServerStore { return &serverStore{db} } type serverStore struct { - *bolt.DB + *sqlx.DB } func (db *serverStore) Find(ctx context.Context, name string) (*autoscaler.Server, error) { - key := []byte(name) - val := new(autoscaler.Server) - err := db.DB.View(func(tx *bolt.Tx) error { - data := tx.Bucket(serverKey).Get(key) - if len(data) == 0 { - return autoscaler.ErrServerNotFound - } - return json.Unmarshal(data, val) - }) - return val, err + dest := &autoscaler.Server{Name: name} + stmt, args, err := db.BindNamed(serverFindStmt, dest) + if err != nil { + return nil, err + } + err = db.GetContext(ctx, dest, stmt, args...) + return dest, err } func (db *serverStore) List(ctx context.Context) ([]*autoscaler.Server, error) { - items := []*autoscaler.Server{} - err := db.DB.View(func(tx *bolt.Tx) error { - c := tx.Bucket(serverKey).Cursor() - for k, v := c.First(); k != nil; k, v = c.Next() { - item := new(autoscaler.Server) - json.Unmarshal(v, item) - items = append(items, item) - } - return nil - }) - return items, err + dest := []*autoscaler.Server{} + err := db.SelectContext(ctx, &dest, serverListStmt) + return dest, err +} + +func (db *serverStore) ListState(ctx context.Context, state autoscaler.ServerState) ([]*autoscaler.Server, error) { + dest := []*autoscaler.Server{} + stmt, args, err := db.BindNamed(serverListStateStmt, map[string]interface{}{"server_state": state}) + if err != nil { + return nil, err + } + err = db.SelectContext(ctx, &dest, stmt, args...) + if err == sql.ErrNoRows { + return dest, nil + } + return dest, err } func (db *serverStore) Create(ctx context.Context, server *autoscaler.Server) error { - return db.Update(ctx, server) + server.Created = time.Now().Unix() + server.Updated = time.Now().Unix() + stmt, args, err := db.BindNamed(serverInsertStmt, server) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, stmt, args...) + return err } func (db *serverStore) Update(ctx context.Context, server *autoscaler.Server) error { - key := []byte(server.Name) - val, _ := json.Marshal(server) - return db.DB.Update(func(tx *bolt.Tx) error { - return tx.Bucket(serverKey).Put(key, val) - }) + // before := server.Updated + server.Updated = time.Now().Unix() + stmt, args, err := db.BindNamed(serverUpdateStmt, server) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, stmt, args...) + return err } func (db *serverStore) Delete(ctx context.Context, server *autoscaler.Server) error { - key := []byte(server.Name) - return db.DB.Update(func(tx *bolt.Tx) error { - return tx.Bucket(serverKey).Delete(key) - }) + stmt, args, err := db.BindNamed(serverDeleteStmt, server) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, stmt, args...) + return err } + +func (db *serverStore) Purge(ctx context.Context, before int64) error { + stmt, args, err := db.BindNamed(serverPurgeStmt, &autoscaler.Server{Stopped: before}) + if err != nil { + return err + } + _, err = db.ExecContext(ctx, stmt, args...) + return err +} + +const serverFindStmt = ` +SELECT + server_name +,server_id +,server_provider +,server_state +,server_image +,server_region +,server_size +,server_address +,server_capacity +,server_secret +,server_error +,server_created +,server_updated +,server_started +,server_stopped +FROM servers +WHERE server_name=:server_name +` + +const serverListStmt = ` +SELECT + server_name +,server_id +,server_provider +,server_state +,server_image +,server_region +,server_size +,server_address +,server_capacity +,server_secret +,server_error +,server_created +,server_updated +,server_started +,server_stopped +FROM servers +ORDER BY server_created ASC +` + +const serverListStateStmt = ` +SELECT + server_name +,server_id +,server_provider +,server_state +,server_image +,server_region +,server_size +,server_address +,server_capacity +,server_secret +,server_error +,server_created +,server_updated +,server_started +,server_stopped +FROM servers +WHERE server_state=:server_state +ORDER BY server_created ASC +` + +const serverInsertStmt = ` +INSERT INTO servers ( + server_name +,server_id +,server_provider +,server_state +,server_image +,server_region +,server_size +,server_address +,server_capacity +,server_secret +,server_error +,server_created +,server_updated +,server_started +,server_stopped +) VALUES ( + :server_name +,:server_id +,:server_provider +,:server_state +,:server_image +,:server_region +,:server_size +,:server_address +,:server_capacity +,:server_secret +,:server_error +,:server_created +,:server_updated +,:server_started +,:server_stopped +) +` + +const serverUpdateStmt = ` +UPDATE servers SET + server_id=:server_id +,server_provider=:server_provider +,server_state=:server_state +,server_image=:server_image +,server_region=:server_region +,server_size=:server_size +,server_address=:server_address +,server_capacity=:server_capacity +,server_secret=:server_secret +,server_error=:server_error +,server_updated=:server_updated +,server_started=:server_started +,server_stopped=:server_stopped +WHERE server_name=:server_name +` + +const serverDeleteStmt = ` +DELETE FROM servers WHERE server_name=:server_name +` + +const serverPurgeStmt = ` +DELETE FROM servers +WHERE server_state = 'stopped' + AND server_stopped < :server_stopped +` diff --git a/store/servers_test.go b/store/servers_test.go index 9a0dedd0..dcc8400a 100644 --- a/store/servers_test.go +++ b/store/servers_test.go @@ -6,7 +6,7 @@ package store import ( "context" - "os" + "database/sql" "testing" "time" @@ -14,24 +14,28 @@ import ( ) func TestServer(t *testing.T) { - temp := tempfile() - defer os.Remove(temp) - - t.Logf("create boltdb database %s", temp) + conn, err := connect() + if err != nil { + t.Error(err) + return + } + defer conn.Close() - db := Must(temp) - store := NewServerStore(db).(*serverStore) + store := NewServerStore(conn).(*serverStore) t.Run("Create", testServerCreate(store)) t.Run("Find", testServerFind(store)) t.Run("List", testServerList(store)) + t.Run("ListState", testServerListState(store)) t.Run("Update", testServerUpdate(store)) t.Run("Delete", testServerDelete(store)) + t.Run("Purge", testServerPurge(store)) } func testServerCreate(store *serverStore) func(t *testing.T) { return func(t *testing.T) { server := &autoscaler.Server{ Provider: autoscaler.ProviderGoogle, + State: autoscaler.StateRunning, Name: "i-5203422c", Address: "54.194.252.215", Capacity: 2, @@ -71,6 +75,32 @@ func testServerList(store *serverStore) func(t *testing.T) { } } +func testServerListState(store *serverStore) func(t *testing.T) { + return func(t *testing.T) { + // seed the database with two servers with shutdown state. + // to confirm we can list servers by state. These will be + // used in a subsequent purge test. + store.Create(context.TODO(), &autoscaler.Server{ + Provider: autoscaler.ProviderGoogle, + State: autoscaler.StateStopped, + Name: "agent-123456789", + }) + store.Create(context.TODO(), &autoscaler.Server{ + Provider: autoscaler.ProviderGoogle, + State: autoscaler.StateStopped, + Name: "agent-987654321", + }) + servers, err := store.ListState(context.TODO(), autoscaler.StateStopped) + if err != nil { + t.Error(err) + return + } + if got, want := len(servers), 2; got != want { + t.Errorf("Want server count %d, got %d", want, got) + } + } +} + func testServerUpdate(store *serverStore) func(t *testing.T) { return func(t *testing.T) { server := &autoscaler.Server{ @@ -112,8 +142,37 @@ func testServerDelete(store *serverStore) func(t *testing.T) { } _, err = store.Find(context.TODO(), "i-5203422c") - if got, want := err, autoscaler.ErrServerNotFound; got != want { - t.Errorf("Want ErrServerNotFound, got %s", got) + if got, want := err, sql.ErrNoRows; got != want { + t.Errorf("Want ErrNoRows, got %s", got) + } + } +} + +func testServerPurge(store *serverStore) func(t *testing.T) { + return func(t *testing.T) { + // this test attempts to purge the database of all + // servers with a state of stopped. The database was + // seeded with stopped servers in testServerListState. + before, _ := store.List(context.TODO()) + if got, want := len(before), 2; got != want { + t.Errorf("Want %d servers, got %d", want, got) + return + } + + err := store.Purge(context.TODO(), time.Now().Unix()+1) + if err != nil { + t.Error(err) + return + } + + after, err := store.List(context.TODO()) + if err != nil { + t.Error(err) + return + } + + if got, want := len(after), 0; got != want { + t.Errorf("Want 0 remaining servers, got %d", got) } } } @@ -123,6 +182,9 @@ func testServer(server *autoscaler.Server) func(t *testing.T) { if got, want := server.Name, "i-5203422c"; got != want { t.Errorf("Want server Name %q, got %q", want, got) } + if got, want := server.State, autoscaler.StateRunning; got != want { + t.Errorf("Want server State %v, got %v", want, got) + } if got, want := server.Address, "54.194.252.215"; got != want { t.Errorf("Want server Address %q, got %q", want, got) } diff --git a/store/store.go b/store/store.go deleted file mode 100644 index d905032c..00000000 --- a/store/store.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package store - -import ( - "github.com/boltdb/bolt" -) - -var serverKey = []byte("servers") - -// New returns a new bolt database connection. -func New(path string) (*bolt.DB, error) { - db, err := bolt.Open(path, 0600, nil) - if err != nil { - return nil, err - } - db.Update(func(tx *bolt.Tx) error { - tx.CreateBucketIfNotExists(serverKey) - return nil - }) - return db, err -} - -// Must returns the bolt database connection and -// panics on error. -func Must(path string) *bolt.DB { - db, err := New(path) - if err != nil { - panic(err) - } - return db -} diff --git a/store/store_test.go b/store/store_test.go deleted file mode 100644 index 20df3c6c..00000000 --- a/store/store_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2018 Drone.IO Inc -// Use of this software is governed by the Business Source License -// that can be found in the LICENSE file. - -package store - -import ( - "io/ioutil" - "os" -) - -// tempfile returns a temporary file path. -func tempfile() string { - f, err := ioutil.TempFile("", "bolt-") - if err != nil { - panic(err) - } - if err := f.Close(); err != nil { - panic(err) - } - if err := os.Remove(f.Name()); err != nil { - panic(err) - } - return f.Name() -}