upgrade/downgrade Windows Agent via uninstall -> install (#29555)

pkg/fleet/installer/service/datadog_agent_windows.go

@@ -10,6 +10,7 @@ package service
 
 import (
 	"context"
+	"fmt"
 	"github.com/DataDog/datadog-agent/pkg/fleet/installer/repository"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/cdn"
 	"github.com/DataDog/datadog-agent/pkg/util/log"
@@ -31,7 +32,7 @@ func SetupAgent(ctx context.Context, args []string) (err error) {
 	}()
 	// Make sure there are no Agent already installed
 	_ = removeProduct("Datadog Agent")
-	err = msiexec("stable", datadogAgent, "/i", args)
+	err = installAgentPackage("stable", args)
 	return err
 }
 
@@ -44,8 +45,18 @@ func StartAgentExperiment(ctx context.Context) (err error) {
 		}
 		span.Finish(tracer.WithError(err))
 	}()
-	err = msiexec("experiment", datadogAgent, "/i", nil)
-	return err
+
+	err = removeAgentIfInstalled(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = installAgentPackage("experiment", nil)
+	if err != nil {
+		// experiment failed, expect stop-experiment to restore the stable Agent
+		return err
+	}
+	return nil
 }
 
 // StopAgentExperiment stops the agent experiment, i.e. removes/uninstalls it.
@@ -57,14 +68,19 @@ func StopAgentExperiment(ctx context.Context) (err error) {
 		}
 		span.Finish(tracer.WithError(err))
 	}()
-	err = msiexec("experiment", datadogAgent, "/x", nil)
+
+	err = removeAgentIfInstalled(ctx)
 	if err != nil {
 		return err
 	}
 
-	// TODO: Need args here to restore DDAGENTUSER
-	err = msiexec("stable", datadogAgent, "/i", nil)
-	return err
+	err = installAgentPackage("stable", nil)
+	if err != nil {
+		// if we cannot restore the stable Agent, the system is left without an Agent
+		return err
+	}
+
+	return nil
 }
 
 // PromoteAgentExperiment promotes the agent experiment
@@ -78,6 +94,8 @@ func RemoveAgent(ctx context.Context) (err error) {
 	span, _ := tracer.StartSpanFromContext(ctx, "remove_agent")
 	defer func() {
 		if err != nil {
+			// removal failed, this should rarely happen.
+			// Rollback might have restored the Agent, but we can't be sure.
 			log.Errorf("Failed to remove agent: %s", err)
 		}
 		span.Finish(tracer.WithError(err))
@@ -90,3 +108,24 @@ func RemoveAgent(ctx context.Context) (err error) {
 func ConfigureAgent(_ context.Context, _ *cdn.CDN, _ *repository.Repositories) error {
 	return nil
 }
+
+func installAgentPackage(target string, args []string) error {
+	// TODO: Need args here to restore DDAGENTUSER
+	err := msiexec(target, datadogAgent, "/i", args)
+	if err != nil {
+		return fmt.Errorf("failed to install Agent %s: %w", target, err)
+	}
+	return nil
+}
+
+func removeAgentIfInstalled(ctx context.Context) error {
+	if isProductInstalled("Datadog Agent") {
+		err := RemoveAgent(ctx)
+		if err != nil {
+			return err
+		}
+	} else {
+		log.Debugf("Agent not installed")
+	}
+	return nil
+}

pkg/fleet/installer/service/msiexec.go

@@ -10,6 +10,7 @@ package service
 import (
 	"fmt"
 	"github.com/DataDog/datadog-agent/pkg/fleet/internal/paths"
+	"github.com/DataDog/datadog-agent/pkg/util/log"
 	"golang.org/x/sys/windows/registry"
 	"os/exec"
 	"path/filepath"
@@ -84,6 +85,7 @@ func processKey(rootPath, key, name string) (*Product, error) {
 // reflect the installed version, and using those installers can lead to undefined behavior (either failure to uninstall,
 // or weird bugs from uninstalling a product with an installer from a different version).
 func removeProduct(productName string) error {
+	log.Debugf("Removing product %s", productName)
 	product, err := findProductCode(productName)
 	if err != nil {
 		return fmt.Errorf("error trying to find product %s: %w", productName, err)
@@ -94,3 +96,11 @@ func removeProduct(productName string) error {
 	}
 	return fmt.Errorf("product %s not found", productName)
 }
+
+func isProductInstalled(productName string) bool {
+	product, err := findProductCode(productName)
+	if err != nil {
+		return false
+	}
+	return product != nil
+}

test/new-e2e/tests/installer/windows/suites/agent-package/upgrade_test.go

@@ -38,6 +38,84 @@ func (s *testAgentUpgradeSuite) TestUpgradeAgentPackage() {
 	})
 }
 
+// TestDowngradeAgentPackage tests that it's possible to downgrade the Datadog Agent using the Datadog installer.
+func (s *testAgentUpgradeSuite) TestDowngradeAgentPackage() {
+	// Arrange
+	_, err := s.Installer().InstallPackage(installerwindows.AgentPackage)
+	s.Require().NoErrorf(err, "failed to install the stable Datadog Agent package")
+
+	// Act
+	_, err = s.Installer().InstallExperiment(installerwindows.AgentPackage,
+		installer.WithRegistry("public.ecr.aws/datadog"),
+		installer.WithVersion(s.StableAgentVersion().PackageVersion()),
+		installer.WithAuthentication(""),
+	)
+
+	// Assert
+	s.Require().NoErrorf(err, "failed to downgrade to stable Datadog Agent package")
+	s.Require().Host(s.Env().RemoteHost).
+		HasARunningDatadogAgentService().
+		WithVersionMatchPredicate(func(version string) {
+			s.Require().Contains(version, s.StableAgentVersion().Version())
+		}).
+		DirExists(installerwindows.GetStableDirFor(installerwindows.AgentPackage))
+}
+
+func (s *testAgentUpgradeSuite) TestExperimentFailure() {
+	// Arrange
+	s.Run("Install stable", func() {
+		s.installStableAgent()
+	})
+
+	// Act
+	_, err := s.Installer().InstallExperiment(installerwindows.AgentPackage,
+		installer.WithRegistry("public.ecr.aws/datadog"),
+		installer.WithVersion("unknown-version"),
+		installer.WithAuthentication(""),
+	)
+
+	// Assert
+	s.Require().Error(err, "expected an error when trying to start an experiment with an unknown version")
+	s.stopExperiment()
+	// TODO: is this the same test as TestStopWithoutExperiment?
+}
+
+func (s *testAgentUpgradeSuite) TestExperimentCurrentVersion() {
+	// Arrange
+	s.Run("Install stable", func() {
+		s.installStableAgent()
+	})
+
+	// Act
+	_, err := s.Installer().InstallExperiment(installerwindows.AgentPackage,
+		installer.WithRegistry("public.ecr.aws/datadog"),
+		installer.WithVersion(s.StableAgentVersion().PackageVersion()),
+		installer.WithAuthentication(""),
+	)
+
+	// Assert
+	s.Require().Error(err, "expected an error when trying to start an experiment with the same version as the current one")
+	s.Require().Host(s.Env().RemoteHost).
+		HasARunningDatadogAgentService().
+		WithVersionMatchPredicate(func(version string) {
+			s.Require().Contains(version, s.StableAgentVersion().Version())
+		}).
+		DirExists(installerwindows.GetStableDirFor(installerwindows.AgentPackage))
+}
+
+func (s *testAgentUpgradeSuite) TestStopWithoutExperiment() {
+	// Arrange
+	s.Run("Install stable", func() {
+		s.installStableAgent()
+	})
+
+	// Act
+
+	// Assert
+	s.stopExperiment()
+	// TODO: Currently uninstalls stable then reinstalls stable. functional but a waste.
+}
+
 func (s *testAgentUpgradeSuite) installStableAgent() {
 	// Arrange