[builder] Support for --skip-new-go-module

.chloggen/builder-skip-go-mod.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: builder
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add a --skip-new-go-module flag to skip creating a module in the output directory.
+
+# One or more tracking issues or pull requests related to the change
+issues: [9252]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/builder/Makefile

@@ -1,5 +1,7 @@
 include ../../Makefile.Common
 
+GOTEST_TIMEOUT=360s
+
 .PHONY: ocb
 ocb:
 	CGO_ENABLED=0 $(GOCMD) build -trimpath -o ../../bin/ocb_$(GOOS)_$(GOARCH) .

cmd/builder/README.md

@@ -157,6 +157,24 @@ ocb --skip-generate --skip-get-modules --config=config.yaml
 ```
 to only execute the compilation step.
 
+### Avoiding the use of a new go.mod file
+
+You can optionally skip creating a new `go.mod` file. This is helpful when 
+using a monorepo setup with a shared go.mod file. When the `--skip-new-go-module` 
+command-line flag is supplied, the build process issues a `go get` command for 
+each component, relying on the Go toolchain to update the enclosing Go module 
+appropriately.
+
+This command will avoid downgrading a dependency in the enclosing
+module, according to
+[`semver.Compare()`](https://pkg.go.dev/golang.org/x/mod/semver#Compare),
+and will instead issue a log indicating that the component was not
+upgraded.
+
+`--skip-new-go-module` is incompatible with `replaces`, `excludes`,
+and the component `path` override.  For each of these features, users
+are expected to modify the enclosing `go.mod` directly.
+
 ### Strict versioning checks
 
 The builder checks the relevant `go.mod`

cmd/builder/go.mod

@@ -31,7 +31,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

cmd/builder/internal/builder/config.go

@@ -19,8 +19,12 @@ import (
 
 const defaultOtelColVersion = "0.107.0"
 
-// ErrMissingGoMod indicates an empty gomod field
-var ErrMissingGoMod = errors.New("missing gomod specification for module")
+var (
+	// ErrMissingGoMod indicates an empty gomod field
+	ErrMissingGoMod = errors.New("missing gomod specification for module")
+	// ErrIncompatibleConfigurationValues indicates that there is configuration that cannot be combined
+	ErrIncompatibleConfigurationValues = errors.New("cannot combine configuration values")
+)
 
 // Config holds the builder's configuration
 type Config struct {
@@ -29,6 +33,7 @@ type Config struct {
 	SkipGenerate         bool   `mapstructure:"-"`
 	SkipCompilation      bool   `mapstructure:"-"`
 	SkipGetModules       bool   `mapstructure:"-"`
+	SkipNewGoModule      bool   `mapstructure:"-"`
 	SkipStrictVersioning bool   `mapstructure:"-"`
 	LDFlags              string `mapstructure:"-"`
 	Verbose              bool   `mapstructure:"-"`
@@ -116,14 +121,15 @@ func NewDefaultConfig() Config {
 func (c *Config) Validate() error {
 	var providersError error
 	if c.Providers != nil {
-		providersError = validateModules("provider", *c.Providers)
+		providersError = c.validateModules("provider", *c.Providers)
 	}
 	return multierr.Combine(
-		validateModules("extension", c.Extensions),
-		validateModules("receiver", c.Receivers),
-		validateModules("exporter", c.Exporters),
-		validateModules("processor", c.Processors),
-		validateModules("connector", c.Connectors),
+		c.validateModules("extension", c.Extensions),
+		c.validateModules("receiver", c.Receivers),
+		c.validateModules("exporter", c.Exporters),
+		c.validateModules("processor", c.Processors),
+		c.validateModules("connector", c.Connectors),
+		c.validateFlags(),
 		providersError,
 	)
 }
@@ -240,11 +246,21 @@ func (c *Config) ParseModules() error {
 	return nil
 }
 
-func validateModules(name string, mods []Module) error {
+func (c *Config) validateFlags() error {
+	if c.SkipNewGoModule && (len(c.Replaces) != 0 || len(c.Excludes) != 0) {
+		return fmt.Errorf("%w excludes or replaces with --skip-new-go-module; please modify the enclosing go.mod file directly", ErrIncompatibleConfigurationValues)
+	}
+	return nil
+}
+
+func (c *Config) validateModules(name string, mods []Module) error {
 	for i, mod := range mods {
 		if mod.GoMod == "" {
 			return fmt.Errorf("%s module at index %v: %w", name, i, ErrMissingGoMod)
 		}
+		if mod.Path != "" && c.SkipNewGoModule {
+			return fmt.Errorf("%w cannot modify mod.path %q combined with --skip-new-go-module; please modify the enclosing go.mod file directly", ErrIncompatibleConfigurationValues, mod.Path)
+		}
 	}
 	return nil
 }

cmd/builder/internal/builder/config_test.go

@@ -4,7 +4,6 @@
 package builder
 
 import (
-	"errors"
 	"os"
 	"strings"
 	"testing"
@@ -140,10 +139,37 @@ func TestMissingModule(t *testing.T) {
 			},
 			err: ErrMissingGoMod,
 		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Extensions: []Module{{
+					GoMod: "some-module",
+					Path:  "invalid",
+				}},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Replaces:        []string{"", ""},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
+		{
+			cfg: Config{
+				Logger:          zap.NewNop(),
+				SkipNewGoModule: true,
+				Excludes:        []string{"", ""},
+			},
+			err: ErrIncompatibleConfigurationValues,
+		},
 	}
 
 	for _, test := range configurations {
-		assert.True(t, errors.Is(test.cfg.Validate(), test.err))
+		assert.ErrorIs(t, test.cfg.Validate(), test.err)
 	}
 }
 

cmd/builder/internal/builder/main.go

@@ -13,7 +13,6 @@
 	"slices"
 	"strings"
 	"text/template"
-	"time"
 
 	"go.uber.org/zap"
 	"golang.org/x/mod/modfile"
@@ -25,7 +24,6 @@
 	ErrGoNotFound      = errors.New("go binary not found")
 	ErrDepNotFound     = errors.New("dependency not found in go mod file")
 	ErrVersionMismatch = errors.New("mismatch in go.mod and builder configuration versions")
-	errDownloadFailed  = errors.New("failed to download go modules")
 	errCompileFailed   = errors.New("failed to compile the OpenTelemetry Collector distribution")
 	skipStrictMsg      = "Use --skip-strict-versioning to temporarily disable this check. This flag will be removed in a future minor version"
 )
@@ -86,18 +84,30 @@
 		return fmt.Errorf("failed to create output path: %w", err)
 	}
 
-	for _, tmpl := range []*template.Template{
+	allTemplates := []*template.Template{
 		mainTemplate,
 		mainOthersTemplate,
 		mainWindowsTemplate,
 		componentsTemplate,
-		goModTemplate,
-	} {
+	}
+
+	// Add the go.mod template unless that file is skipped.
+	if !cfg.SkipNewGoModule {
+		allTemplates = append(allTemplates, goModTemplate)
+	}
+
+	for _, tmpl := range allTemplates {
 		if err := processAndWrite(cfg, tmpl, tmpl.Name(), cfg); err != nil {
 			return fmt.Errorf("failed to generate source file %q: %w", tmpl.Name(), err)
 		}
 	}
 
+	// when not creating a new go.mod file, update modules one-by-one in the
+	// enclosing go module.
+	if err := cfg.updateModules(); err != nil {
+		return err
+	}
+
 	cfg.Logger.Info("Sources created", zap.String("path", cfg.Distribution.OutputPath))
 	return nil
 }
@@ -145,7 +155,7 @@
 	}
 
 	if cfg.SkipStrictVersioning {
-		return downloadModules(cfg)
+		return nil
 	}
 
 	// Perform strict version checking.  For each component listed and the
@@ -185,22 +195,7 @@
 		}
 	}
 
-	return downloadModules(cfg)
-}
-
-func downloadModules(cfg Config) error {
-	cfg.Logger.Info("Getting go modules")
-	failReason := "unknown"
-	for i := 1; i <= cfg.downloadModules.numRetries; i++ {
-		if _, err := runGoCommand(cfg, "mod", "download"); err != nil {
-			failReason = err.Error()
-			cfg.Logger.Info("Failed modules download", zap.String("retry", fmt.Sprintf("%d/%d", i, cfg.downloadModules.numRetries)))
-			time.Sleep(cfg.downloadModules.wait)
-			continue
-		}
-		return nil
-	}
-	return fmt.Errorf("%w: %s", errDownloadFailed, failReason)
+	return nil
 }
 
 func processAndWrite(cfg Config, tmpl *template.Template, outFile string, tmplParams any) error {
@@ -226,6 +221,68 @@
 		c.Extensions, c.Connectors, *c.Providers)
 }
 
+func (c *Config) updateModules() error {
+	if !c.SkipNewGoModule {
+		return nil
+	}
+
+	// Build the main service dependency
+	coremod, corever := c.coreModuleAndVersion()
+	corespec := coremod + " " + corever
+
+	if err := c.updateGoModule(corespec); err != nil {
+		return err
+	}
+
+	for _, comp := range c.allComponents() {
+		if err := c.updateGoModule(comp.GoMod); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *Config) updateGoModule(modspec string) error {
+	mod, ver, found := strings.Cut(modspec, " ")
+	if !found {
+		return fmt.Errorf("ill-formatted modspec %q: missing space separator", modspec)
+	}
+
+	// Re-parse the go.mod file on each iteration, since it can
+	// change each time.
+	modulePath, dependencyVersions, err := c.readGoModFile()
+	if err != nil {
+		return err
+	}
+
+	if mod == modulePath {
+		// this component is part of the same module, nothing to update.
+		return nil
+	}
+
+	// check for exact match
+	hasVer, ok := dependencyVersions[mod]
+	if ok && hasVer == ver {
+		c.Logger.Info("Component version match", zap.String("module", mod), zap.String("version", ver))
+		return nil
+	}
+
+	scomp := semver.Compare(hasVer, ver)
+	if scomp > 0 {
+		// version in enclosing module is newer, do not change
+		c.Logger.Info("Not upgrading component, enclosing module is newer.", zap.String("module", mod), zap.String("existing", hasVer), zap.String("config_version", ver))
+		return nil
+	}
+
+	// upgrading or changing version
+	updatespec := "-require=" + mod + "@" + ver
+
+	if _, err := runGoCommand(*c, "mod", "edit", updatespec); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *Config) readGoModFile() (string, map[string]string, error) {
 	var modPath string
 	stdout, err := runGoCommand(*c, "mod", "edit", "-print")