From 20cb3e396b37b89a57009d7b90341b1dce59526b Mon Sep 17 00:00:00 2001 From: Chao Dai Date: Fri, 2 Jul 2021 10:53:48 -0700 Subject: [PATCH] Prow pubsub: supports presubmit jobs --- prow/config/config.go | 7 ++- prow/pubsub/subscriber/subscriber.go | 84 ++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/prow/config/config.go b/prow/config/config.go index a2011be1ca851..213e83283ddb1 100644 --- a/prow/config/config.go +++ b/prow/config/config.go @@ -361,7 +361,12 @@ func (c *Config) GetPresubmits(gc git.ClientFactory, identifier string, baseSHAG return nil, err } - return append(c.PresubmitsStatic[identifier], prowYAML.Presubmits...), nil + return append(c.GetPresubmitsStatic(identifier), prowYAML.Presubmits...), nil +} + +// GetPresubmitsStatic will return presubmits for the given identifier that are versioned inside the tested repo +func (c *Config) GetPresubmitsStatic(identifier string) []Presubmit { + return c.PresubmitsStatic[identifier] } // GetPostsubmits will return all postsubmits for the given identifier. This includes diff --git a/prow/pubsub/subscriber/subscriber.go b/prow/pubsub/subscriber/subscriber.go index 586502233db6e..afcec6473d2de 100644 --- a/prow/pubsub/subscriber/subscriber.go +++ b/prow/pubsub/subscriber/subscriber.go @@ -33,6 +33,8 @@ import ( prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1" v1 "k8s.io/test-infra/prow/apis/prowjobs/v1" "k8s.io/test-infra/prow/config" + "k8s.io/test-infra/prow/git/v2" + "k8s.io/test-infra/prow/github" "k8s.io/test-infra/prow/pjutil" ) @@ -43,6 +45,22 @@ const ( postsubmitProwJobEvent = "prow.k8s.io/pubsub.PostsubmitProwJobEvent" ) +// Ensure interface is intact +var _ prowCfgClient = (*config.Config)(nil) + +type githubClient interface { + GetRef(org, repo, ref string) (string, error) + // Compatible with config.githubClient + GetPullRequestChanges(org, repo string, number int) ([]github.PullRequestChange, error) +} + +// prowCfgClient is for unit test purpose +type prowCfgClient interface { + AllPeriodics() []config.Periodic + GetPresubmits(gc git.ClientFactory, identifier string, baseSHAGetter config.RefGetter, headSHAGetters ...config.RefGetter) ([]config.Presubmit, error) + GetPresubmitsStatic(identifier string) []config.Presubmit +} + // ProwJobEvent contains the minimum information required to start a ProwJob. type ProwJobEvent struct { Name string `json:"name"` @@ -129,13 +147,13 @@ func (m *pubSubMessage) nack() { // jobHandler handles job type specific logic type jobHandler interface { - getProwJobSpec(cfg *config.Config, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) + getProwJobSpec(cfg prowCfgClient, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) } // periodicJobHandler implements jobHandler type periodicJobHandler struct{} -func (peh *periodicJobHandler) getProwJobSpec(cfg *config.Config, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) { +func (peh *periodicJobHandler) getProwJobSpec(cfg prowCfgClient, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) { var periodicJob *config.Periodic // TODO(chaodaiG): do we want to support inrepoconfig when // https://github.com/kubernetes/test-infra/issues/21729 is done? @@ -154,16 +172,70 @@ func (peh *periodicJobHandler) getProwJobSpec(cfg *config.Config, pe ProwJobEven } // presubmitJobHandler implements jobHandler -type presubmitJobHandler struct{} +type presubmitJobHandler struct { + GitHubClient githubClient + GitClient git.ClientFactory +} + +func (prh *presubmitJobHandler) getProwJobSpec(cfg prowCfgClient, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) { + // presubmit jobs require Refs and Refs.Pulls to be set + refs := pe.Refs + if refs == nil { + return nil, nil, errors.New("Refs must be supplied") + } + if len(refs.Pulls) != 1 { + return nil, nil, fmt.Errorf("expect 1 pull, got: %d", len(refs.Pulls)) + } + pr := refs.Pulls[0] + + var presubmitJob *config.Presubmit + org, repo, branch := refs.Org, refs.Repo, pr.Ref + orgRepo := org + "/" + repo + baseSHAGetter := func() (string, error) { + if refs.BaseSHA != "" { + return refs.BaseSHA, nil + } + // Fall back to fetch from GitHub + baseSHA, err := prh.GitHubClient.GetRef(refs.Org, refs.Repo, "heads/"+refs.BaseRef) + if err != nil { + return "", fmt.Errorf("failed to get baseSHA: %v", err) + } + return baseSHA, nil + } + headSHAGetter := func() (string, error) { + return pr.SHA, nil + } + + // This will work with inrepoconfig + presubmits, err := cfg.GetPresubmits(prh.GitClient, orgRepo, baseSHAGetter, headSHAGetter) + if err != nil { + // Fall back to static presubmits to avoid deadlocking when a presubmit is used to verify + // inrepoconfig + logrus.WithError(err).Debug("Failed to get presubmits") + presubmits = cfg.GetPresubmitsStatic(orgRepo) + } + + for _, job := range presubmits { + if !job.CouldRun(branch) { // filter out jobs that are not branch matching + continue + } + if job.Name == pe.Name { + presubmitJob = &job // running multiple jobs with the same name for a PR is not a good idea, so take the first one + break + } + } + if presubmitJob == nil { + return nil, nil, fmt.Errorf("failed to find associated periodic job %q", pe.Name) + } -func (prh *presubmitJobHandler) getProwJobSpec(cfg *config.Config, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) { - return nil, nil, errors.New("presubmit not supported yet") + prowJobSpec := pjutil.PresubmitSpec(*presubmitJob, *refs) + return &prowJobSpec, presubmitJob.Labels, nil } // ppostsubmitJobHandler implements jobHandler type postsubmitJobHandler struct{} -func (poh *postsubmitJobHandler) getProwJobSpec(cfg *config.Config, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) { +func (poh *postsubmitJobHandler) getProwJobSpec(cfg prowCfgClient, pe ProwJobEvent) (*v1.ProwJobSpec, map[string]string, error) { return nil, nil, errors.New("postsubmit not supported yet") }