Skip to content

Commit

Permalink
test: BlueGreenStrategy should be tested (#3922)
Browse files Browse the repository at this point in the history
This deployment strategy is pretty complex and caused some surprises
before.
  • Loading branch information
kzys authored Sep 10, 2024
1 parent 74f1b8c commit 99e170b
Show file tree
Hide file tree
Showing 3 changed files with 310 additions and 8 deletions.
184 changes: 184 additions & 0 deletions internal/command/deploy/mock_client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
package deploy

import (
"context"
"fmt"
"net/http"
"time"

fly "github.com/superfly/fly-go"
)

type mockWebClient struct {
}

func (f *mockWebClient) CanPerformBluegreenDeployment(ctx context.Context, appName string) (bool, error) {
return true, nil
}

type mockFlapsClient struct {
breakLaunch bool
breakWait bool
breakUncordon bool
breakSetMetadata bool
}

func (m *mockFlapsClient) AcquireLease(ctx context.Context, machineID string, ttl *int) (*fly.MachineLease, error) {
return nil, fmt.Errorf("failed to acquire lease for %s", machineID)
}

func (m *mockFlapsClient) Cordon(ctx context.Context, machineID string, nonce string) (err error) {
return fmt.Errorf("failed to cordon %s", machineID)
}

func (m *mockFlapsClient) CreateApp(ctx context.Context, name string, org string) (err error) {
return fmt.Errorf("failed to create app %s", name)
}

func (m *mockFlapsClient) CreateVolume(ctx context.Context, req fly.CreateVolumeRequest) (*fly.Volume, error) {
return nil, fmt.Errorf("failed to create volume %s", req.Name)
}

func (m *mockFlapsClient) CreateVolumeSnapshot(ctx context.Context, volumeId string) error {
return fmt.Errorf("failed to create volume snapshot %s", volumeId)
}

func (m *mockFlapsClient) DeleteMetadata(ctx context.Context, machineID, key string) error {
return fmt.Errorf("failed to delete metadata %s", key)
}

func (m *mockFlapsClient) DeleteVolume(ctx context.Context, volumeId string) (*fly.Volume, error) {
return nil, fmt.Errorf("failed to delete volume %s", volumeId)
}

func (m *mockFlapsClient) Destroy(ctx context.Context, input fly.RemoveMachineInput, nonce string) (err error) {
return fmt.Errorf("failed to destroy %s", input.ID)
}

func (m *mockFlapsClient) Exec(ctx context.Context, machineID string, in *fly.MachineExecRequest) (*fly.MachineExecResponse, error) {
return nil, fmt.Errorf("failed to exec %s", machineID)
}

func (m *mockFlapsClient) ExtendVolume(ctx context.Context, volumeId string, size_gb int) (*fly.Volume, bool, error) {
return nil, false, fmt.Errorf("failed to extend volume %s", volumeId)
}

func (m *mockFlapsClient) FindLease(ctx context.Context, machineID string) (*fly.MachineLease, error) {
return nil, fmt.Errorf("failed to find lease for %s", machineID)
}

func (m *mockFlapsClient) Get(ctx context.Context, machineID string) (*fly.Machine, error) {
return nil, fmt.Errorf("failed to get %s", machineID)
}

func (m *mockFlapsClient) GetAllVolumes(ctx context.Context) ([]fly.Volume, error) {
return nil, fmt.Errorf("failed to get all volumes")
}

func (m *mockFlapsClient) GetMany(ctx context.Context, machineIDs []string) ([]*fly.Machine, error) {
return nil, fmt.Errorf("failed to get machines")
}

func (m *mockFlapsClient) GetMetadata(ctx context.Context, machineID string) (map[string]string, error) {
return nil, fmt.Errorf("failed to get metadata for %s", machineID)
}

func (m *mockFlapsClient) GetProcesses(ctx context.Context, machineID string) (fly.MachinePsResponse, error) {
return nil, fmt.Errorf("failed to get processes for %s", machineID)
}

func (m *mockFlapsClient) GetVolume(ctx context.Context, volumeId string) (*fly.Volume, error) {
return nil, fmt.Errorf("failed to get volume %s", volumeId)
}

func (m *mockFlapsClient) GetVolumeSnapshots(ctx context.Context, volumeId string) ([]fly.VolumeSnapshot, error) {
return nil, fmt.Errorf("failed to get volume snapshots for %s", volumeId)
}

func (m *mockFlapsClient) GetVolumes(ctx context.Context) ([]fly.Volume, error) {
return nil, fmt.Errorf("failed to get volumes")
}

func (m *mockFlapsClient) Kill(ctx context.Context, machineID string) (err error) {
return fmt.Errorf("failed to kill %s", machineID)
}

func (m *mockFlapsClient) Launch(ctx context.Context, builder fly.LaunchMachineInput) (*fly.Machine, error) {
if m.breakLaunch {
return nil, fmt.Errorf("failed to launch %s", builder.ID)
}
return &fly.Machine{}, nil
}

func (m *mockFlapsClient) List(ctx context.Context, state string) ([]*fly.Machine, error) {
return nil, fmt.Errorf("failed to list machines")
}

func (m *mockFlapsClient) ListActive(ctx context.Context) ([]*fly.Machine, error) {
return nil, fmt.Errorf("failed to list active machines")
}

func (m *mockFlapsClient) ListFlyAppsMachines(ctx context.Context) ([]*fly.Machine, *fly.Machine, error) {
return nil, nil, fmt.Errorf("failed to list fly apps machines")
}

func (m *mockFlapsClient) NewRequest(ctx context.Context, method, path string, in interface{}, headers map[string][]string) (*http.Request, error) {
return nil, fmt.Errorf("failed to create request")
}

func (m *mockFlapsClient) RefreshLease(ctx context.Context, machineID string, ttl *int, nonce string) (*fly.MachineLease, error) {
return nil, fmt.Errorf("failed to refresh lease for %s", machineID)
}

func (m *mockFlapsClient) ReleaseLease(ctx context.Context, machineID, nonce string) error {
return fmt.Errorf("failed to release lease for %s", machineID)
}

func (m *mockFlapsClient) Restart(ctx context.Context, in fly.RestartMachineInput, nonce string) (err error) {
return fmt.Errorf("failed to restart %s", in.ID)
}

func (m *mockFlapsClient) SetMetadata(ctx context.Context, machineID, key, value string) error {
if m.breakSetMetadata {
return fmt.Errorf("failed to set metadata for %s", machineID)
}
return nil
}

func (m *mockFlapsClient) Start(ctx context.Context, machineID string, nonce string) (out *fly.MachineStartResponse, err error) {
return nil, fmt.Errorf("failed to start %s", machineID)
}

func (m *mockFlapsClient) Stop(ctx context.Context, in fly.StopMachineInput, nonce string) (err error) {
return fmt.Errorf("failed to stop %s", in.ID)
}

func (m *mockFlapsClient) Suspend(ctx context.Context, machineID, nonce string) (err error) {
return fmt.Errorf("failed to suspend %s", machineID)
}

func (m *mockFlapsClient) Uncordon(ctx context.Context, machineID string, nonce string) (err error) {
if m.breakUncordon {
return fmt.Errorf("failed to uncordon %s", machineID)
}
return nil
}

func (m *mockFlapsClient) Update(ctx context.Context, builder fly.LaunchMachineInput, nonce string) (out *fly.Machine, err error) {
return nil, fmt.Errorf("failed to update %s", builder.ID)
}

func (m *mockFlapsClient) UpdateVolume(ctx context.Context, volumeId string, req fly.UpdateVolumeRequest) (*fly.Volume, error) {
return nil, fmt.Errorf("failed to update volume %s", volumeId)
}

func (m *mockFlapsClient) Wait(ctx context.Context, machine *fly.Machine, state string, timeout time.Duration) (err error) {
if m.breakWait {
return fmt.Errorf("failed to wait for %s", machine.ID)
}
return nil
}

func (m *mockFlapsClient) WaitForApp(ctx context.Context, name string) error {
return fmt.Errorf("failed to wait for app %s", name)
}
23 changes: 18 additions & 5 deletions internal/command/deploy/strategy_bluegreen.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"github.com/superfly/flyctl/internal/appconfig"
"github.com/superfly/flyctl/internal/ctrlc"
"github.com/superfly/flyctl/internal/flapsutil"
"github.com/superfly/flyctl/internal/flyutil"
"github.com/superfly/flyctl/internal/machine"
"github.com/superfly/flyctl/internal/tracing"
"github.com/superfly/flyctl/iostreams"
Expand Down Expand Up @@ -57,11 +56,15 @@ type RollbackLog struct {
disableRollback bool
}

type blueGreenWebClient interface {
CanPerformBluegreenDeployment(ctx context.Context, appName string) (bool, error)
}

type blueGreen struct {
greenMachines machineUpdateEntries
blueMachines machineUpdateEntries
flaps flapsutil.FlapsClient
apiClient flyutil.Client
apiClient blueGreenWebClient
io *iostreams.IOStreams
colorize *iostreams.ColorScheme
clearLinesAbove func(count int)
Expand All @@ -77,6 +80,9 @@ type blueGreen struct {
maxConcurrent int

rollbackLog RollbackLog

waitBeforeStop time.Duration
waitBeforeCordon time.Duration
}

func BlueGreenStrategy(md *machineDeployment, blueMachines []*machineUpdateEntry) *blueGreen {
Expand All @@ -100,13 +106,20 @@ func BlueGreenStrategy(md *machineDeployment, blueMachines []*machineUpdateEntry
rollbackLog: RollbackLog{canDeleteGreenMachines: true, disableRollback: false},
}

bg.initialize()

return bg
}

func (bg *blueGreen) initialize() {
// Hook into Ctrl+C so that we can rollback the deployment when it's aborted.
ctrlc.ClearHandlers()
bg.ctrlcHook = ctrlc.Hook(sync.OnceFunc(func() {
close(bg.aborted)
}))

return bg
bg.waitBeforeStop = 10 * time.Second
bg.waitBeforeCordon = 10 * time.Second
}

func (bg *blueGreen) isAborted() bool {
Expand Down Expand Up @@ -757,7 +770,7 @@ func (bg *blueGreen) Deploy(ctx context.Context) error {

// Wait a bit to let fly-proxy see the new machines
fmt.Fprintf(bg.io.ErrOut, "\nWaiting before cordoning all blue machines\n")
if bg.sleepAbortable(10 * time.Second) {
if bg.sleepAbortable(bg.waitBeforeCordon) {
return ErrAborted
}

Expand All @@ -773,7 +786,7 @@ func (bg *blueGreen) Deploy(ctx context.Context) error {

// Wait a bit to let fly-proxy forget about the old machines
fmt.Fprintf(bg.io.ErrOut, "\nWaiting before stopping all blue machines\n")
if bg.sleepAbortable(10 * time.Second) {
if bg.sleepAbortable(bg.waitBeforeStop) {
return ErrAborted
}

Expand Down
111 changes: 108 additions & 3 deletions internal/command/deploy/strategy_bluegreen_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,117 @@
package deploy

import (
"context"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/superfly/fly-go"
"github.com/superfly/flyctl/internal/appconfig"
"github.com/superfly/flyctl/internal/flapsutil"
"github.com/superfly/flyctl/internal/machine"
"github.com/superfly/flyctl/iostreams"
)

func TestBlueGreenStrategy(t *testing.T) {
s := BlueGreenStrategy(&machineDeployment{}, nil)
assert.False(t, s.isAborted())
func TestNew(t *testing.T) {
strategy := BlueGreenStrategy(&machineDeployment{}, nil)
assert.False(t, strategy.isAborted())
}

func newBlueGreenStrategy(client flapsutil.FlapsClient, numberOfExistingMachines int) *blueGreen {
var machines []*machineUpdateEntry
ios, _, _, _ := iostreams.Test()

for i := 0; i < numberOfExistingMachines; i++ {
machines = append(machines, &machineUpdateEntry{
leasableMachine: machine.NewLeasableMachine(client, ios, &fly.Machine{}, false),
launchInput: &fly.LaunchMachineInput{
Config: &fly.MachineConfig{
Metadata: map[string]string{},
Checks: map[string]fly.MachineCheck{
"check1": {},
},
},
},
})
}
strategy := &blueGreen{
apiClient: &mockWebClient{},
flaps: client,
maxConcurrent: 10,
appConfig: &appconfig.Config{},
io: ios,
colorize: ios.ColorScheme(),
timeout: 1 * time.Second,
blueMachines: machines,
}
strategy.initialize()

// Don't have to wait during tests.
strategy.waitBeforeStop = 0
strategy.waitBeforeCordon = 0

return strategy
}

func TestDeploy(t *testing.T) {
flapsClient := &mockFlapsClient{}

ctx := context.Background()

// Some functions take a client from the context.
ctx = flapsutil.NewContextWithClient(ctx, flapsClient)

// Happy cases
t.Run("replace 1 machine", func(t *testing.T) {
flapsClient.breakLaunch = false
strategy := newBlueGreenStrategy(flapsClient, 1)

err := strategy.Deploy(ctx)
assert.NoError(t, err)
})
t.Run("replace 10 machine", func(t *testing.T) {
flapsClient.breakLaunch = false
strategy := newBlueGreenStrategy(flapsClient, 10)

err := strategy.Deploy(ctx)
assert.NoError(t, err)
})

// Error cases
t.Run("no existing machines", func(t *testing.T) {
strategy := newBlueGreenStrategy(flapsClient, 0)

err := strategy.Deploy(ctx)
assert.ErrorContains(t, err, "found multiple image versions")
})
t.Run("failed to launch machines", func(t *testing.T) {
flapsClient.breakLaunch = true
strategy := newBlueGreenStrategy(flapsClient, 1)

err := strategy.Deploy(ctx)
assert.ErrorContains(t, err, "failed to create green machines")
})
}

func FuzzDeploy(f *testing.F) {
flapsClient := &mockFlapsClient{}

ctx := context.Background()

// Some functions take a client from the context.
ctx = flapsutil.NewContextWithClient(ctx, flapsClient)

f.Add(20, false, false, false, false)

f.Fuzz(func(t *testing.T, numberOfExistingMachines int, breakLaunch bool, breakWait bool, breakUncordon bool, breakSetMetadata bool) {
strategy := newBlueGreenStrategy(flapsClient, numberOfExistingMachines)
flapsClient.breakLaunch = breakLaunch
flapsClient.breakWait = breakWait
flapsClient.breakUncordon = breakUncordon
flapsClient.breakSetMetadata = breakSetMetadata

// At least, Deploy must not panic.
strategy.Deploy(ctx)
})
}

0 comments on commit 99e170b

Please sign in to comment.