[builder] Support for --skip-new-go-module (#10098)

A continuation of
#9253 and
#9631

Description: Adds a `--skip-new-go-module` flag to the OTC builder. This
enables users working in an existing go module environment (say, a
"monorepo") to update the module they have, vs forcing the use of a new
module.

With the new support inside an existing Go module, a collector main
package can be generated using a go:generate directive. For example, in
the directory where I want my collector built, the file generate.go has
this line:

//go:generate builder --skip-new-go-module --skip-compilation
--strict-versioning --config=./build-config.yaml
In the same directory, the build-config.yaml describes the collector to
build. The builder generates the other files in the same directory. At
this point, normal Go workflows can be used to update indirect
dependencies.

Link to tracking Issue:
#9252

Testing: Will add unit tests in the next few days.

Documentation: Yes.

---------

Co-authored-by: Pablo Baeyens <pbaeyens31+github@gmail.com>

.chloggen/builder-skip-go-mod.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: builder
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add a --skip-new-go-module flag to skip creating a module in the output directory.
+
+# One or more tracking issues or pull requests related to the change
+issues: [9252]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/builder/Makefile

@@ -1,5 +1,7 @@
 include ../../Makefile.Common
 
+GOTEST_TIMEOUT=360s
+
 .PHONY: ocb
 ocb:
 	CGO_ENABLED=0 $(GOCMD) build -trimpath -o ../../bin/ocb_$(GOOS)_$(GOARCH) .

cmd/builder/README.md

@@ -157,6 +157,24 @@ ocb --skip-generate --skip-get-modules --config=config.yaml
 ```
 to only execute the compilation step.
 
+### Avoiding the use of a new go.mod file
+
+You can optionally skip creating a new `go.mod` file. This is helpful when 
+using a monorepo setup with a shared go.mod file. When the `--skip-new-go-module` 
+command-line flag is supplied, the build process issues a `go get` command for 
+each component, relying on the Go toolchain to update the enclosing Go module 
+appropriately.
+
+This command will avoid downgrading a dependency in the enclosing
+module, according to
+[`semver.Compare()`](https://pkg.go.dev/golang.org/x/mod/semver#Compare),
+and will instead issue a log indicating that the component was not
+upgraded.
+
+`--skip-new-go-module` is incompatible with `replaces`, `excludes`,
+and the component `path` override.  For each of these features, users
+are expected to modify the enclosing `go.mod` directly.
+
 ### Strict versioning checks
 
 The builder checks the relevant `go.mod`

cmd/builder/go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

cmd/builder/internal/builder/config.go

@@ -19,8 +19,12 @@ import (
 
 const defaultOtelColVersion = "0.107.0"
 
-// ErrMissingGoMod indicates an empty gomod field
-var ErrMissingGoMod = errors.New("missing gomod specification for module")
+var (
+	// ErrMissingGoMod indicates an empty gomod field
+	ErrMissingGoMod = errors.New("missing gomod specification for module")
+	// ErrIncompatibleConfigurationValues indicates that there is configuration that cannot be combined
+	ErrIncompatibleConfigurationValues = errors.New("cannot combine configuration values")
+)
 
 // Config holds the builder's configuration
 type Config struct {
@@ -29,6 +33,7 @@ type Config struct {
 	SkipGenerate         bool   `mapstructure:"-"`
 	SkipCompilation      bool   `mapstructure:"-"`
 	SkipGetModules       bool   `mapstructure:"-"`
+	SkipNewGoModule      bool   `mapstructure:"-"`
 	SkipStrictVersioning bool   `mapstructure:"-"`
 	LDFlags              string `mapstructure:"-"`
 	Verbose              bool   `mapstructure:"-"`
@@ -116,14 +121,15 @@ func NewDefaultConfig() Config {
 func (c *Config) Validate() error {
 	var providersError error
 	if c.Providers != nil {
-		providersError = validateModules("provider", *c.Providers)
+		providersError = c.validateModules("provider", *c.Providers)
 	}
 	return multierr.Combine(
-		validateModules("extension", c.Extensions),
-		validateModules("receiver", c.Receivers),
-		validateModules("exporter", c.Exporters),
-		validateModules("processor", c.Processors),
-		validateModules("connector", c.Connectors),
+		c.validateModules("extension", c.Extensions),
+		c.validateModules("receiver", c.Receivers),
+		c.validateModules("exporter", c.Exporters),
+		c.validateModules("processor", c.Processors),
+		c.validateModules("connector", c.Connectors),
+		c.validateFlags(),
 		providersError,
 	)
 }
@@ -240,11 +246,21 @@ func (c *Config) ParseModules() error {
 	return nil
 }
 
-func validateModules(name string, mods []Module) error {
+func (c *Config) validateFlags() error {
+	if c.SkipNewGoModule && (len(c.Replaces) != 0 || len(c.Excludes) != 0) {
+		return fmt.Errorf("%w excludes or replaces with --skip-new-go-module; please modify the enclosing go.mod file directly", ErrIncompatibleConfigurationValues)
+	}
+	return nil
+}
+
+func (c *Config) validateModules(name string, mods []Module) error {
 	for i, mod := range mods {
 		if mod.GoMod == "" {
 			return fmt.Errorf("%s module at index %v: %w", name, i, ErrMissingGoMod)
 		}
+		if mod.Path != "" && c.SkipNewGoModule {
+			return fmt.Errorf("%w cannot modify mod.path %q combined with --skip-new-go-module; please modify the enclosing go.mod file directly", ErrIncompatibleConfigurationValues, mod.Path)
+		}
 	}
 	return nil
 }

cmd/builder/internal/builder/config_test.go

@@ -4,7 +4,6 @@
 package builder
 
 import (
-	"errors"
 	"os"
 	"strings"
 	"testing"
@@ -140,10 +139,37 @@ func TestMissingModule(t *testing.T) {
 			},
 			err: ErrMissingGoMod,
 		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Extensions: []Module{{
+					GoMod: "some-module",
+					Path:  "invalid",
+				}},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Replaces:        []string{"", ""},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Excludes:        []string{"", ""},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
 	}
 
 	for _, test := range configurations {
-		assert.True(t, errors.Is(test.cfg.Validate(), test.err))
+		assert.ErrorIs(t, test.cfg.Validate(), test.err)
 	}
 }
 

cmd/builder/internal/builder/main.go

@@ -13,7 +13,6 @@ import (
 	"slices"
 	"strings"
 	"text/template"
-	"time"
 
 	"go.uber.org/zap"
 	"golang.org/x/mod/modfile"
@@ -25,7 +24,6 @@ var (
 	ErrGoNotFound      = errors.New("go binary not found")
 	ErrDepNotFound     = errors.New("dependency not found in go mod file")
 	ErrVersionMismatch = errors.New("mismatch in go.mod and builder configuration versions")
-	errDownloadFailed  = errors.New("failed to download go modules")
 	errCompileFailed   = errors.New("failed to compile the OpenTelemetry Collector distribution")
 	skipStrictMsg      = "Use --skip-strict-versioning to temporarily disable this check. This flag will be removed in a future minor version"
 )
@@ -86,18 +84,30 @@ func Generate(cfg Config) error {
 		return fmt.Errorf("failed to create output path: %w", err)
 	}
 
-	for _, tmpl := range []*template.Template{
+	allTemplates := []*template.Template{
 		mainTemplate,
 		mainOthersTemplate,
 		mainWindowsTemplate,
 		componentsTemplate,
-		goModTemplate,
-	} {
+	}
+
+	// Add the go.mod template unless that file is skipped.
+	if !cfg.SkipNewGoModule {
+		allTemplates = append(allTemplates, goModTemplate)
+	}
+
+	for _, tmpl := range allTemplates {
 		if err := processAndWrite(cfg, tmpl, tmpl.Name(), cfg); err != nil {
 			return fmt.Errorf("failed to generate source file %q: %w", tmpl.Name(), err)
 		}
 	}
 
+	// when not creating a new go.mod file, update modules one-by-one in the
+	// enclosing go module.
+	if err := cfg.updateModules(); err != nil {
+		return err
+	}
+
 	cfg.Logger.Info("Sources created", zap.String("path", cfg.Distribution.OutputPath))
 	return nil
 }
@@ -145,7 +155,7 @@ func GetModules(cfg Config) error {
 	}
 
 	if cfg.SkipStrictVersioning {
-		return downloadModules(cfg)
+		return nil
 	}
 
 	// Perform strict version checking.  For each component listed and the
@@ -185,22 +195,7 @@ func GetModules(cfg Config) error {
 		}
 	}
 
-	return downloadModules(cfg)
-}
-
-func downloadModules(cfg Config) error {
-	cfg.Logger.Info("Getting go modules")
-	failReason := "unknown"
-	for i := 1; i <= cfg.downloadModules.numRetries; i++ {
-		if _, err := runGoCommand(cfg, "mod", "download"); err != nil {
-			failReason = err.Error()
-			cfg.Logger.Info("Failed modules download", zap.String("retry", fmt.Sprintf("%d/%d", i, cfg.downloadModules.numRetries)))
-			time.Sleep(cfg.downloadModules.wait)
-			continue
-		}
-		return nil
-	}
-	return fmt.Errorf("%w: %s", errDownloadFailed, failReason)
+	return nil
 }
 
 func processAndWrite(cfg Config, tmpl *template.Template, outFile string, tmplParams any) error {
@@ -226,6 +221,68 @@ func (c *Config) allComponents() []Module {
 		c.Extensions, c.Connectors, *c.Providers)
 }
 
+func (c *Config) updateModules() error {
+	if !c.SkipNewGoModule {
+		return nil
+	}
+
+	// Build the main service dependency
+	coremod, corever := c.coreModuleAndVersion()
+	corespec := coremod + " " + corever
+
+	if err := c.updateGoModule(corespec); err != nil {
+		return err
+	}
+
+	for _, comp := range c.allComponents() {
+		if err := c.updateGoModule(comp.GoMod); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Config) updateGoModule(modspec string) error {
+	mod, ver, found := strings.Cut(modspec, " ")
+	if !found {
+		return fmt.Errorf("ill-formatted modspec %q: missing space separator", modspec)
+	}
+
+	// Re-parse the go.mod file on each iteration, since it can
+	// change each time.
+	modulePath, dependencyVersions, err := c.readGoModFile()
+	if err != nil {
+		return err
+	}
+
+	if mod == modulePath {
+		// this component is part of the same module, nothing to update.
+		return nil
+	}
+
+	// check for exact match
+	hasVer, ok := dependencyVersions[mod]
+	if ok && hasVer == ver {
+		c.Logger.Info("Component version match", zap.String("module", mod), zap.String("version", ver))
+		return nil
+	}
+
+	scomp := semver.Compare(hasVer, ver)
+	if scomp > 0 {
+		// version in enclosing module is newer, do not change
+		c.Logger.Info("Not upgrading component, enclosing module is newer.", zap.String("module", mod), zap.String("existing", hasVer), zap.String("config_version", ver))
+		return nil
+	}
+
+	// upgrading or changing version
+	updatespec := "-require=" + mod + "@" + ver
+
+	if _, err := runGoCommand(*c, "mod", "edit", updatespec); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *Config) readGoModFile() (string, map[string]string, error) {
 	var modPath string
 	stdout, err := runGoCommand(*c, "mod", "edit", "-print")