Add docker image validation (gardener#154)

* Fix tm bot deployment

* Add docker image check

* Add better error reporting

pkg/common/common.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Copyright (c) 2019 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+const (
+	DockerImageGardenerApiServer = "eu.gcr.io/gardener-project/gardener/apiserver"
+)
+
+// Repositories
+const (
+	TestInfraRepo   = "https://github.com/gardener/test-infra.git"
+	GardenSetupRepo = "https://github.com/schrodit/garden-setup.git"
+	GardenerRepo    = "https://github.com/gardener/gardener.git"
+)

pkg/testrunner/renderer/templates/templates.go

@@ -16,15 +16,10 @@ package templates
 
 import (
 	"github.com/gardener/test-infra/pkg/apis/testmachinery/v1beta1"
+	"github.com/gardener/test-infra/pkg/common"
 	"github.com/gardener/test-infra/pkg/hostscheduler"
 )
 
-const (
-	TestInfraRepo   = "https://github.com/gardener/test-infra.git"
-	GardenSetupRepo = "https://github.com/schrodit/garden-setup.git"
-	GardenerRepo    = "https://github.com/gardener/gardener.git"
-)
-
 var TestInfraLocationName = "tm"
 var DefaultLocationSetName = "default"
 
@@ -34,7 +29,7 @@ var TestInfraLocation = v1beta1.LocationSet{
 	Locations: []v1beta1.TestLocation{
 		{
 			Type:     v1beta1.LocationTypeGit,
-			Repo:     TestInfraRepo,
+			Repo:     common.TestInfraRepo,
 			Revision: "master",
 		},
 	},
@@ -51,7 +46,7 @@ func GetDefaultLocationsSet(cfg GardenerConfig) v1beta1.LocationSet {
 		set.Locations = []v1beta1.TestLocation{
 			{
 				Type:     v1beta1.LocationTypeGit,
-				Repo:     GardenerRepo,
+				Repo:     common.GardenerRepo,
 				Revision: cfg.Version,
 			},
 		}
@@ -60,7 +55,7 @@ func GetDefaultLocationsSet(cfg GardenerConfig) v1beta1.LocationSet {
 		set.Locations = []v1beta1.TestLocation{
 			{
 				Type:     v1beta1.LocationTypeGit,
-				Repo:     GardenerRepo,
+				Repo:     common.GardenerRepo,
 				Revision: cfg.Commit,
 			},
 		}
@@ -77,7 +72,7 @@ func GetGardenSetupLocation(name, revision string) v1beta1.LocationSet {
 		Locations: []v1beta1.TestLocation{
 			{
 				Type:     v1beta1.LocationTypeGit,
-				Repo:     GardenSetupRepo,
+				Repo:     common.GardenSetupRepo,
 				Revision: revision,
 			},
 		},

pkg/tm-bot/github/client.go

@@ -56,7 +56,11 @@ func (c *client) ResolveConfigValue(event *GenericRequestEvent, value *ghval.Git
 		return pr.GetHead().GetSHA(), nil
 	}
 	if value.Path != nil {
-		file, dir, _, err := c.client.Repositories.GetContents(context.TODO(), event.GetOwnerName(), event.GetRepositoryName(), *value.Path, &github.RepositoryContentGetOptions{Ref: event.Repository.GetDefaultBranch()})
+		pr, _, err := c.client.PullRequests.Get(context.TODO(), event.GetOwnerName(), event.GetRepositoryName(), event.Number)
+		if err != nil {
+			return "", err
+		}
+		file, dir, _, err := c.client.Repositories.GetContents(context.TODO(), event.GetOwnerName(), event.GetRepositoryName(), *value.Path, &github.RepositoryContentGetOptions{Ref: pr.GetHead().GetSHA()})
 		if err != nil {
 			return "", err
 		}

pkg/tm-bot/plugins/errors/errors.go

@@ -74,5 +74,14 @@ func ShortForError(err error) string {
 	case *PluginError:
 		return t.short
 	}
-	return "Unkown error"
+	return "Unknown error"
+}
+
+// LongForError returns long message for the error
+func LongForError(err error) string {
+	switch t := err.(type) {
+	case *PluginError:
+		return t.long
+	}
+	return "Unknown error"
 }

pkg/tm-bot/plugins/request.go

@@ -30,32 +30,32 @@ func HandleRequest(client github.Client, event *github.GenericRequestEvent) erro
 	}
 
 	for _, args := range commands {
-		go runPlugin(client, event, args)
+		go Plugins.runPlugin(client, event, args)
 	}
 
 	return nil
 }
 
 // runPlugin runs a plugin with a event and its arguments
-func runPlugin(client github.Client, event *github.GenericRequestEvent, args []string) {
-	runID, plugin, err := Plugins.Get(args[0])
+func (p *plugins) runPlugin(client github.Client, event *github.GenericRequestEvent, args []string) {
+	runID, plugin, err := p.Get(args[0])
 	if err != nil {
-		_ = Error(client, event, err)
+		_ = p.Error(client, event, nil, err)
 		return
 	}
 
-	Plugins.initState(plugin, runID, event)
+	p.initState(plugin, runID, event)
 
 	fs := plugin.Flags()
 	if err := fs.Parse(args[1:]); err != nil {
-		Plugins.RemoveState(plugin, runID)
-		_ = Error(client, event, pluginerr.New(err.Error(), FormatUsageError(args[0], plugin.Description(), plugin.Example(), fs.FlagUsages())))
+		p.RemoveState(plugin, runID)
+		_ = p.Error(client, event, plugin, pluginerr.New(err.Error(), "unable to parse flags"))
 		return
 	}
 	if err := plugin.Run(fs, client, event); err != nil {
 		if !pluginerr.IsRecoverable(err) {
 			Plugins.RemoveState(plugin, runID)
-			_ = Error(client, event, pluginerr.New(err.Error(), FormatUsageError(args[0], plugin.Description(), plugin.Example(), fs.FlagUsages())))
+			_ = p.Error(client, event, plugin, err)
 		}
 		return
 	}
@@ -71,7 +71,7 @@ func (p *plugins) resumePlugin(ghMgr github.Manager, name, runID string, state *
 		return
 	}
 
-	_, plugin, err := Plugins.Get(name)
+	_, plugin, err := p.Get(name)
 	if err != nil {
 		p.log.Error(err, "unable to get plugin for state", "plugin", name)
 		return
@@ -80,20 +80,24 @@ func (p *plugins) resumePlugin(ghMgr github.Manager, name, runID string, state *
 
 	if err := plugin.ResumeFromState(ghClient, state.Event, state.Custom); err != nil {
 		if !pluginerr.IsRecoverable(err) {
-			Plugins.RemoveState(plugin, runID)
-			_ = Error(ghClient, state.Event, pluginerr.New(err.Error(), FormatUsageError(name, plugin.Description(), plugin.Example(), plugin.Flags().FlagUsages())))
+			p.RemoveState(plugin, runID)
+			_ = p.Error(ghClient, state.Event, plugin, err)
 		}
 		return
 	}
 	Plugins.RemoveState(plugin, runID)
 }
 
 // Error responds to the client if an error occurs
-func Error(client github.Client, event *github.GenericRequestEvent, err error) error {
-	if _, err := client.Comment(event, FormatErrorResponse(event.GetAuthorName(), pluginerr.ShortForError(err), err.Error())); err != nil {
+func (p *plugins) Error(client github.Client, event *github.GenericRequestEvent, plugin Plugin, err error) error {
+	p.log.Error(err, err.Error())
+
+	if plugin != nil {
+		_, err := client.Comment(event, FormatErrorResponse(event.GetAuthorName(), pluginerr.ShortForError(err), FormatUsageError(plugin.Command(), plugin.Description(), plugin.Example(), plugin.Flags().FlagUsages())))
 		return err
 	}
-	return nil
+	_, err = client.Comment(event, FormatSimpleErrorResponse(event.GetAuthorName(), pluginerr.ShortForError(err)))
+	return err
 }
 
 // ParseCommands parses a message and returns a string of commands and arguments

pkg/tm-bot/plugins/respond.go

@@ -45,16 +45,28 @@ func FormatResponseWithReason(to, message, reason string) string {
 	return fmt.Sprintf(format, to, message, reason, AboutThisBotWithoutCommands)
 }
 
+// FormatSimpleErrorResponse formats a response that does not warrant additional explanation in the
+// details section.
+func FormatSimpleErrorResponse(to, message string) string {
+	format := `:fire: Oops, something went wrong @%s
+%s
+
+>%s`
+
+	return fmt.Sprintf(format, to, message, AboutThisBotWithoutCommands)
+}
+
 // FormatErrorResponse formats a response that does not warrant additional explanation in the
 // details section.
 func FormatErrorResponse(to, message, reason string) string {
-	format := `:fire: Oops, something went wrong
-@%s: %s
+	format := `:fire: Oops, something went wrong @%s
+%s
 <details>
 %s
 
-%s
-</details>`
+</details>
+
+>%s`
 
 	return fmt.Sprintf(format, to, message, reason, AboutThisBotWithoutCommands)
 }

pkg/tm-bot/plugins/tests/flags.go

@@ -15,10 +15,14 @@
 package tests
 
 import (
+	"fmt"
 	"github.com/gardener/gardener/pkg/apis/garden/v1beta1"
+	"github.com/gardener/test-infra/pkg/common"
 	"github.com/gardener/test-infra/pkg/hostscheduler/gardenerscheduler"
 	"github.com/gardener/test-infra/pkg/testmachinery"
 	"github.com/gardener/test-infra/pkg/tm-bot/github"
+	"github.com/gardener/test-infra/pkg/tm-bot/plugins/errors"
+	"github.com/gardener/test-infra/pkg/util"
 	"github.com/gardener/test-infra/pkg/util/cmdvalues"
 	"github.com/ghodss/yaml"
 	"github.com/spf13/pflag"
@@ -34,7 +38,13 @@ const (
 	cloudprovider       = "cloudprovider"
 )
 
-func (t *test) ValidateFlags(flagset *pflag.FlagSet) error {
+func (t *test) ValidateConfig() error {
+	if t.config.Gardener.Version != "" {
+		if err := util.CheckDockerImageExists(common.DockerImageGardenerApiServer, t.config.Gardener.Version); err != nil {
+			return errors.New(fmt.Sprintf("I am unable to find gardener images of version %s.\n Have you specified the right version?", t.config.Gardener.Version),
+				"Maybe you should run the default gardener pipeline before trying to run the integration tests.")
+		}
+	}
 	return nil
 }
 
@@ -61,16 +71,16 @@ func (t *test) Flags() *pflag.FlagSet {
 	return flagset
 }
 
-func (t *test) ApplyDefaultConfig(client github.Client, event *github.GenericRequestEvent, flagset *pflag.FlagSet) {
+func (t *test) ApplyDefaultConfig(client github.Client, event *github.GenericRequestEvent, flagset *pflag.FlagSet) error {
 	raw, err := client.GetConfig(t.Command())
 	if err != nil {
 		t.log.Error(err, "cannot get default config")
-		return
+		return nil
 	}
 	var defaultConfig DefaultsConfig
 	if err := yaml.Unmarshal(raw, &defaultConfig); err != nil {
 		t.log.Error(err, "unable to parse default config")
-		return
+		return errors.New("unable to parse default config", err.Error())
 	}
 
 	if !flagset.Changed(hostprovider) && defaultConfig.HostProvider != nil {
@@ -82,23 +92,23 @@ func (t *test) ApplyDefaultConfig(client github.Client, event *github.GenericReq
 	if !flagset.Changed(gardensetupRevision) && defaultConfig.GardenSetup != nil && defaultConfig.GardenSetup.Revision != nil {
 		val, err := client.ResolveConfigValue(event, defaultConfig.GardenSetup.Revision.Value())
 		if err != nil {
-			t.log.Error(err, "unable to resolve config value for garden setup revision")
+			return errors.New("unable to resolve default config value for garden setup revision", err.Error())
 		} else {
 			t.config.GardenSetupRevision = val
 		}
 	}
 	if !flagset.Changed(gardenerVersion) && defaultConfig.Gardener != nil && defaultConfig.Gardener.Version != nil {
 		val, err := client.ResolveConfigValue(event, defaultConfig.Gardener.Version.Value())
 		if err != nil {
-			t.log.Error(err, "unable to resolve config value for gardener version")
+			return errors.New("unable to resolve default config value for gardener version", err.Error())
 		} else {
 			t.config.Gardener.Version = val
 		}
 	}
 	if !flagset.Changed(gardenerCommit) && defaultConfig.Gardener != nil && defaultConfig.Gardener.Commit != nil {
 		val, err := client.ResolveConfigValue(event, defaultConfig.Gardener.Commit.Value())
 		if err != nil {
-			t.log.Error(err, "unable to resolve config value for gardener commit")
+			return errors.New("unable to resolve default config value for gardener commit", err.Error())
 		} else {
 			t.config.Gardener.Commit = val
 		}
@@ -110,4 +120,5 @@ func (t *test) ApplyDefaultConfig(client github.Client, event *github.GenericReq
 	if !flagset.Changed(cloudprovider) && defaultConfig.CloudProviders != nil {
 		t.config.Shoots.CloudProviders = *defaultConfig.CloudProviders
 	}
+	return nil
 }

pkg/tm-bot/plugins/tests/run.go

@@ -38,7 +38,12 @@ func (t *test) Run(flagset *pflag.FlagSet, client github.Client, event *github.G
 	ctx := context.Background()
 	defer ctx.Done()
 
-	t.ApplyDefaultConfig(client, event, flagset)
+	if err := t.ApplyDefaultConfig(client, event, flagset); err != nil {
+		return err
+	}
+	if err := t.ValidateConfig(); err != nil {
+		return err
+	}
 
 	t.config.Shoots.DefaultTest = templates.TestWithLabels(t.testLabel)
 	if t.hibernation {