From 0db927b38fccc9a6a08c185cb92ce6b546b4c53f Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Tue, 3 Oct 2023 17:57:31 -0700 Subject: [PATCH] [chore] change completely how dependabot generates entries (#27269) **Description:** Change entirely how dependabot update entries are generated, by using the metadata.yaml status to find which components are most important in the distribution. The code now takes into account the distributions and the stability of the component as a score to decide whether to push the component. Go modules that don't have an associated metadata.yaml are not considered and therefore not present in the module updates path. **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/19410 --- .github/dependabot.yml | 190 ++++++++++++++++++++--------------------- Makefile | 25 +----- cmd/githubgen/go.mod | 2 +- cmd/githubgen/main.go | 163 ++++++++++++++++++++++++++++++----- 4 files changed, 241 insertions(+), 139 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e1d93abc9b00..8f95c615262c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,21 +7,6 @@ updates: schedule: interval: "weekly" day: "wednesday" - - package-ecosystem: "gomod" - directory: "/cmd/configschema" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/cmd/githubgen" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/cmd/mdatagen" - schedule: - interval: "weekly" - day: "wednesday" - package-ecosystem: "gomod" directory: "/cmd/opampsupervisor" schedule: @@ -77,16 +62,6 @@ updates: schedule: interval: "weekly" day: "wednesday" - - package-ecosystem: "gomod" - directory: "/examples/demo/client" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/examples/demo/server" - schedule: - interval: "weekly" - day: "wednesday" - package-ecosystem: "gomod" directory: "/exporter/alibabacloudlogserviceexporter" schedule: @@ -337,16 +312,6 @@ updates: schedule: interval: "weekly" day: "wednesday" - - package-ecosystem: "gomod" - directory: "/extension/encoding/jaegerencodingextension" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/extension/encoding/textencodingextension" - schedule: - interval: "weekly" - day: "wednesday" - package-ecosystem: "gomod" directory: "/extension/headerssetterextension" schedule: @@ -422,61 +387,6 @@ updates: schedule: interval: "weekly" day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/awsutil" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/containerinsight" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/cwlogs" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/ecsutil" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/k8s" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/metrics" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/proxy" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/xray" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/xray/testdata/sampleapp" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/aws/xray/testdata/sampleserver" - schedule: - interval: "weekly" - day: "wednesday" - - package-ecosystem: "gomod" - directory: "/internal/common" - schedule: - interval: "weekly" - day: "wednesday" - package-ecosystem: "gomod" directory: "/internal/coreinternal" schedule: @@ -1077,11 +987,6 @@ updates: schedule: interval: "weekly" day: "wednesday" - - package-ecosystem: "gomod" - directory: "/receiver/simpleprometheusreceiver/examples/federation/prom-counter" - schedule: - interval: "weekly" - day: "wednesday" - package-ecosystem: "gomod" directory: "/receiver/skywalkingreceiver" schedule: @@ -1102,3 +1007,98 @@ updates: schedule: interval: "weekly" day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/splunkenterprisereceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/splunkhecreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/sqlqueryreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/sqlserverreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/sshcheckreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/statsdreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/syslogreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/tcplogreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/udplogreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/vcenterreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/wavefrontreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/webhookeventreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/windowseventlogreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/windowsperfcountersreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/zipkinreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/receiver/zookeeperreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/testbed" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/testbed/mockdatareceivers/mockawsxrayreceiver" + schedule: + interval: "weekly" + day: "wednesday" + - package-ecosystem: "gomod" + directory: "/testbed/mockdatasenders/mockdatadogagentexporter" + schedule: + interval: "weekly" + day: "wednesday" diff --git a/Makefile b/Makefile index d09d98cb70cd..5609995d718c 100644 --- a/Makefile +++ b/Makefile @@ -145,29 +145,8 @@ push-tags: $(MULITMOD) DEPENDABOT_PATH=".github/dependabot.yml" .PHONY: gendependabot gendependabot: - @echo "Recreating ${DEPENDABOT_PATH} file" - @echo "# File generated by \"make gendependabot\"; DO NOT EDIT." > ${DEPENDABOT_PATH} - @echo "" >> ${DEPENDABOT_PATH} - @echo "version: 2" >> ${DEPENDABOT_PATH} - @echo "updates:" >> ${DEPENDABOT_PATH} - @echo "Add entry for \"/\" gomod" - @echo " - package-ecosystem: \"gomod\"" >> ${DEPENDABOT_PATH} - @echo " directory: \"/\"" >> ${DEPENDABOT_PATH} - @echo " schedule:" >> ${DEPENDABOT_PATH} - @echo " interval: \"weekly\"" >> ${DEPENDABOT_PATH} - @echo " day: \"wednesday\"" >> ${DEPENDABOT_PATH} - @set -e; for dir in `echo $(NONROOT_MODS) | tr ' ' '\n' | head -n 219 | tr '\n' ' '`; do \ - echo "Add entry for \"$${dir:1}\""; \ - echo " - package-ecosystem: \"gomod\"" >> ${DEPENDABOT_PATH}; \ - echo " directory: \"$${dir:1}\"" >> ${DEPENDABOT_PATH}; \ - echo " schedule:" >> ${DEPENDABOT_PATH}; \ - echo " interval: \"weekly\"" >> ${DEPENDABOT_PATH}; \ - echo " day: \"wednesday\"" >> ${DEPENDABOT_PATH}; \ - done - @echo "The following modules are not included in the dependabot file because it has a limit of 220 entries:" - @set -e; for dir in `echo $(NONROOT_MODS) | tr ' ' '\n' | tail -n +220 | tr '\n' ' '`; do \ - echo " - $${dir:1}"; \ - done + cd cmd/githubgen && $(GOCMD) install . + githubgen -dependabot # Define a delegation target for each module diff --git a/cmd/githubgen/go.mod b/cmd/githubgen/go.mod index bd353b723d6c..845517c6e301 100644 --- a/cmd/githubgen/go.mod +++ b/cmd/githubgen/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/google/go-github/v53 v53.2.0 go.opentelemetry.io/collector/confmap v0.86.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -26,5 +27,4 @@ require ( golang.org/x/sys v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.31.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/cmd/githubgen/main.go b/cmd/githubgen/main.go index 7d7bb41beef8..efc161d72cb3 100644 --- a/cmd/githubgen/main.go +++ b/cmd/githubgen/main.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "context" "flag" "fmt" @@ -14,10 +15,14 @@ import ( "sort" "strings" + "gopkg.in/yaml.v3" + "github.com/google/go-github/v53/github" "go.opentelemetry.io/collector/confmap/provider/fileprovider" ) +const maxDependabotUpdates = 220 + const codeownersHeader = `# Code generated by githubgen. DO NOT EDIT. ##################################################### # @@ -74,20 +79,84 @@ internal/common ` +const dependabotHeader = `# File generated by "make gendependabot"; DO NOT EDIT. + +` + const unmaintainedStatus = "unmaintained" -// Generates files specific to Github according to status metadata: +// Generates files specific to Github or Dependabot according to status metadata: // .github/CODEOWNERS // .github/ALLOWLIST +// .github/dependabot.yml func main() { folder := flag.String("folder", ".", "folder investigated for codeowners") allowlistFilePath := flag.String("allowlist", "cmd/githubgen/allowlist.txt", "path to a file containing an allowlist of members outside the OpenTelemetry organization") + dependabotOnly := flag.Bool("dependabot", false, "only generate dependabot entries") flag.Parse() - if err := run(*folder, *allowlistFilePath); err != nil { + if err := run(*folder, *allowlistFilePath, *dependabotOnly); err != nil { log.Fatal(err) } } +type DependabotSchedule struct { + Interval yaml.Node `yaml:"interval"` + Day yaml.Node `yaml:"day"` +} + +type DependabotUpdate struct { + PackageEcosystem yaml.Node `yaml:"package-ecosystem"` + Directory yaml.Node `yaml:"directory"` + Schedule DependabotSchedule `yaml:"schedule"` + Priority int `yaml:"-"` +} + +type DependabotData struct { + Version int `yaml:"version"` + Updates []DependabotUpdate `yaml:"updates"` +} + +func makePriority(status *Status) int { + // not an internal component such as pkg/**, and no distributions: + if len(status.Distributions) == 0 && status.Class != "" { + return 1 + } + // start with a score of 2 + maxScore := 2 + for stability := range status.Stability { + switch stability { + case "deprecated": // stay with 2 + case "unmaintained": + return 1 + case "alpha": + if maxScore < 3 { + maxScore = 3 + } + case "beta": + if maxScore < 4 { + maxScore = 4 + } + case "stable": + if maxScore < 5 { + maxScore = 5 + } + } + } + return maxScore +} + +func newDependabotUpdate(directory string, priority int) DependabotUpdate { + return DependabotUpdate{ + PackageEcosystem: yaml.Node{Value: "gomod", Style: yaml.DoubleQuotedStyle, Kind: yaml.ScalarNode}, + Directory: yaml.Node{Value: "/" + directory, Style: yaml.DoubleQuotedStyle, Kind: yaml.ScalarNode}, + Schedule: DependabotSchedule{ + Interval: yaml.Node{Value: "weekly", Style: yaml.DoubleQuotedStyle, Kind: yaml.ScalarNode}, + Day: yaml.Node{Value: "wednesday", Style: yaml.DoubleQuotedStyle, Kind: yaml.ScalarNode}, + }, + Priority: priority, + } +} + type Codeowners struct { // Active codeowners Active []string `mapstructure:"active"` @@ -129,27 +198,19 @@ func loadMetadata(filePath string) (metadata, error) { return md, nil } -func run(folder string, allowlistFilePath string) error { - members, err := getGithubMembers() - if err != nil { - return err - } - allowlistData, err := os.ReadFile(allowlistFilePath) - if err != nil { - return err - } - allowlistLines := strings.Split(string(allowlistData), "\n") - - allowlist := make(map[string]struct{}, len(allowlistLines)) - for _, line := range allowlistLines { - allowlist[line] = struct{}{} - } +func run(folder string, allowlistFilePath string, dependabotOnly bool) error { components := map[string]metadata{} var foldersList []string + dependabotData := &DependabotData{ + Version: 2, + Updates: []DependabotUpdate{ + newDependabotUpdate("", 5), + }, + } maxLength := 0 allCodeowners := map[string]struct{}{} - err = filepath.Walk(folder, func(path string, info fs.FileInfo, err error) error { + err := filepath.Walk(folder, func(path string, info fs.FileInfo, err error) error { if info.Name() == "metadata.yaml" { m, err := loadMetadata(path) if err != nil { @@ -158,15 +219,23 @@ func run(folder string, allowlistFilePath string) error { if m.Status == nil { return nil } - key := filepath.Dir(path) + "/" + currentFolder := filepath.Dir(path) + key := currentFolder + "/" components[key] = m foldersList = append(foldersList, key) + _, err = os.Stat(filepath.Join(currentFolder, "go.mod")) + if err == nil { // go.mod file exists. + dependabotData.Updates = append(dependabotData.Updates, newDependabotUpdate(currentFolder, makePriority(m.Status))) + } for stability := range m.Status.Stability { if stability == unmaintainedStatus { // do not account for unmaintained status to change the max length of the component line. return nil } } + if m.Status.Codeowners == nil { + return nil + } for _, id := range m.Status.Codeowners.Active { allCodeowners[id] = struct{}{} } @@ -179,9 +248,64 @@ func run(folder string, allowlistFilePath string) error { if err != nil { return err } + if !dependabotOnly { + if err = generateCodeownersFiles(foldersList, allCodeowners, allowlistFilePath, components, maxLength); err != nil { + return err + } + } + + sort.Slice(dependabotData.Updates, func(i, j int) bool { + return dependabotData.Updates[i].Priority > dependabotData.Updates[j].Priority + }) + removed := dependabotData.Updates[maxDependabotUpdates:] + dependabotData.Updates = dependabotData.Updates[:maxDependabotUpdates] + if len(removed) > 0 { + sort.Slice(removed, func(i, j int) bool { + return strings.Compare(removed[i].Directory.Value, removed[j].Directory.Value) < 0 + }) + fmt.Printf("The following modules were not added to Dependabot to keep within limits of %d\n", maxDependabotUpdates) + for _, update := range removed { + fmt.Printf(" - %q\n", update.Directory.Value) + } + } + + sort.Slice(dependabotData.Updates, func(i, j int) bool { + return strings.Compare(dependabotData.Updates[i].Directory.Value, dependabotData.Updates[j].Directory.Value) < 0 + }) + + var yamlContents bytes.Buffer + encoder := yaml.NewEncoder(&yamlContents) + encoder.SetIndent(2) + err = encoder.Encode(dependabotData) + if err != nil { + return err + } + err = os.WriteFile(filepath.Join(".github", "dependabot.yml"), append([]byte(dependabotHeader), yamlContents.Bytes()...), 0600) + if err != nil { + return err + } + + return nil +} + +func generateCodeownersFiles(foldersList []string, allCodeowners map[string]struct{}, allowlistFilePath string, components map[string]metadata, maxLength int) error { + allowlistData, err := os.ReadFile(allowlistFilePath) + if err != nil { + return err + } + allowlistLines := strings.Split(string(allowlistData), "\n") + + allowlist := make(map[string]struct{}, len(allowlistLines)) + for _, line := range allowlistLines { + allowlist[line] = struct{}{} + } sort.Strings(foldersList) var missingCodeowners []string var duplicateCodeowners []string + members, err := getGithubMembers() + if err != nil { + return err + } for codeowner := range allCodeowners { _, present := members[codeowner] @@ -248,7 +372,6 @@ LOOP: if err != nil { return err } - return nil }