[TUF autoupdater] Check out latest

cmd/launcher/launcher.go

@@ -365,7 +365,7 @@ func runLauncher(ctx context.Context, cancel func(), opts *launcher.Options) err
 		metadataClient := http.DefaultClient
 		metadataClient.Timeout = 1 * time.Minute
 		mirrorClient := http.DefaultClient
-		mirrorClient.Timeout = 5 * time.Minute // gives us extra time to avoid a timeout on download
+		mirrorClient.Timeout = 8 * time.Minute // gives us extra time to avoid a timeout on download
 		tufAutoupdater, err := tuf.NewTufAutoupdater(
 			k,
 			metadataClient,

docs/architecture/library_lookup.md

@@ -0,0 +1,23 @@
+## Library lookup
+
+When launcher looks for the version to run for itself or for osquery, it first
+looks through local TUF metadata to see if it knows what version to run for its
+given release channel. If it does, and the version is already downloaded, it
+will run that version.
+
+Otherwise, it will look for the most recent version downloaded to its update
+library.
+
+```mermaid
+flowchart TB
+    A[Library lookup] --> B{Do we have a local TUF repo?}
+    B ---->|No| C[Get most recent version from update library]
+    C --> D[Return path to most recent version of executable]
+    D --> H[End]
+    B -->|Yes| E{release.json target metadata exists?}
+    E -->|No| C
+    E --> |Yes| F{Target indicated by release.json\nis downloaded to update library?}
+    F --> |Yes| G[Return path to selected executable in update library]
+    F --> |No| C
+    G --> H
+```

pkg/autoupdate/tuf/autoupdate.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -102,7 +103,7 @@ func NewTufAutoupdater(k types.Knapsack, metadataHttpClient *http.Client, mirror
 	// If the update directory wasn't set by a flag, use the default location of <launcher root>/updates.
 	updateDirectory := k.UpdateDirectory()
 	if updateDirectory == "" {
-		updateDirectory = DefaultLibraryDirectory(k.RootDirectory())
+		updateDirectory = defaultLibraryDirectory(k.RootDirectory())
 	}
 	ta.libraryManager, err = newUpdateLibraryManager(k.MirrorServerURL(), mirrorHttpClient, updateDirectory, ta.logger)
 	if err != nil {
@@ -148,7 +149,7 @@ func LocalTufDirectory(rootDirectory string) string {
 	return filepath.Join(rootDirectory, tufDirectoryName)
 }
 
-func DefaultLibraryDirectory(rootDirectory string) string {
+func defaultLibraryDirectory(rootDirectory string) string {
 	return filepath.Join(rootDirectory, "updates")
 }
 
@@ -296,7 +297,7 @@ func (ta *TufAutoupdater) checkForUpdate() error {
 // downloadUpdate will download a new release for the given binary, if available from TUF
 // and not already downloaded.
 func (ta *TufAutoupdater) downloadUpdate(binary autoupdatableBinary, targets data.TargetFiles) (string, error) {
-	release, releaseMetadata, err := ta.findRelease(binary, targets)
+	release, releaseMetadata, err := findRelease(binary, targets, ta.channel)
 	if err != nil {
 		return "", fmt.Errorf("could not find release: %w", err)
 	}
@@ -322,12 +323,12 @@ func (ta *TufAutoupdater) downloadUpdate(binary autoupdatableBinary, targets dat
 }
 
 // findRelease checks the latest data from TUF (in `targets`) to see whether a new release
-// has been published for our channel. If it has, it returns the target for that release
+// has been published for the given channel. If it has, it returns the target for that release
 // and its associated metadata.
-func (ta *TufAutoupdater) findRelease(binary autoupdatableBinary, targets data.TargetFiles) (string, data.TargetFileMeta, error) {
+func findRelease(binary autoupdatableBinary, targets data.TargetFiles, channel string) (string, data.TargetFileMeta, error) {
 	// First, find the target that the channel release file is pointing to
 	var releaseTarget string
-	targetReleaseFile := fmt.Sprintf("%s/%s/%s/release.json", binary, runtime.GOOS, ta.channel)
+	targetReleaseFile := path.Join(string(binary), runtime.GOOS, channel, "release.json")
 	for targetName, target := range targets {
 		if targetName != targetReleaseFile {
 			continue
@@ -357,7 +358,7 @@ func (ta *TufAutoupdater) findRelease(binary autoupdatableBinary, targets data.T
 		return filepath.Base(releaseTarget), target, nil
 	}
 
-	return "", data.TargetFileMeta{}, fmt.Errorf("could not find metadata for release target %s for binary %s", targetReleaseFile, binary)
+	return "", data.TargetFileMeta{}, fmt.Errorf("could not find metadata for release target %s for binary %s", releaseTarget, binary)
 }
 
 // storeError saves errors that occur during the periodic check for updates, so that they

pkg/autoupdate/tuf/channel_version.go

@@ -0,0 +1,35 @@
+package tuf
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/kolide/launcher/pkg/agent"
+)
+
+// GetChannelVersionFromTufServer returns the tagged version of the given binary for the given release channel.
+// It is intended for use in e.g. packaging and `launcher doctor`, where we want to determine the correct
+// version tag for the channel but do not necessarily have access to a local TUF repo.
+func GetChannelVersionFromTufServer(binary, channel, tufServerUrl string) (string, error) {
+	tempTufRepoDir, err := agent.MkdirTemp("temp-tuf")
+	if err != nil {
+		return "", fmt.Errorf("could not make temporary directory: %w", err)
+	}
+
+	tempTufClient, err := initMetadataClient(tempTufRepoDir, tufServerUrl, http.DefaultClient)
+	if err != nil {
+		return "", fmt.Errorf("could not init metadata client: %w", err)
+	}
+
+	targets, err := tempTufClient.Update()
+	if err != nil {
+		return "", fmt.Errorf("could not update targets: %w", err)
+	}
+
+	releaseTarget, _, err := findRelease(autoupdatableBinary(binary), targets, channel)
+	if err != nil {
+		return "", fmt.Errorf("could not find release: %w", err)
+	}
+
+	return versionFromTarget(autoupdatableBinary(binary), releaseTarget), nil
+}

pkg/autoupdate/tuf/library_lookup.go

@@ -0,0 +1,90 @@
+package tuf
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path/filepath"
+
+	"github.com/go-kit/kit/log"
+	"github.com/go-kit/kit/log/level"
+	"github.com/kolide/launcher/pkg/autoupdate"
+)
+
+type BinaryUpdateInfo struct {
+	Path    string
+	Version string
+}
+
+// CheckOutLatest returns the path to the latest downloaded executable for our binary, as well
+// as its version.
+func CheckOutLatest(binary autoupdatableBinary, rootDirectory string, updateDirectory string, channel string, logger log.Logger) (*BinaryUpdateInfo, error) {
+	if updateDirectory == "" {
+		updateDirectory = defaultLibraryDirectory(rootDirectory)
+	}
+
+	update, err := findExecutableFromRelease(binary, LocalTufDirectory(rootDirectory), channel, updateDirectory)
+	if err == nil {
+		return update, nil
+	}
+
+	level.Debug(logger).Log("msg", "could not find executable from release", "err", err)
+
+	// If we can't find the specific release version that we should be on, then just return the executable
+	// with the most recent version in the library
+	return mostRecentVersion(binary, updateDirectory)
+}
+
+// findExecutableFromRelease looks at our local TUF repository to find the release for our
+// given channel. If it's already downloaded, then we return its path and version.
+func findExecutableFromRelease(binary autoupdatableBinary, tufRepositoryLocation string, channel string, baseUpdateDirectory string) (*BinaryUpdateInfo, error) {
+	// Initialize a read-only TUF metadata client to parse the data we already have downloaded about releases.
+	metadataClient, err := readOnlyTufMetadataClient(tufRepositoryLocation)
+	if err != nil {
+		return nil, errors.New("could not initialize TUF client, cannot find release")
+	}
+
+	// From already-downloaded metadata, look for the release version
+	targets, err := metadataClient.Targets()
+	if err != nil {
+		return nil, fmt.Errorf("could not get target: %w", err)
+	}
+
+	targetName, _, err := findRelease(binary, targets, channel)
+	if err != nil {
+		return nil, fmt.Errorf("could not find release: %w", err)
+	}
+
+	targetPath, targetVersion := pathToTargetVersionExecutable(binary, targetName, baseUpdateDirectory)
+	if autoupdate.CheckExecutable(context.TODO(), targetPath, "--version") != nil {
+		return nil, fmt.Errorf("version %s from target %s either not yet downloaded or corrupted: %w", targetVersion, targetName, err)
+	}
+
+	return &BinaryUpdateInfo{
+		Path:    targetPath,
+		Version: targetVersion,
+	}, nil
+}
+
+// mostRecentVersion returns the path to the most recent, valid version available in the library for the
+// given binary, along with its version.
+func mostRecentVersion(binary autoupdatableBinary, baseUpdateDirectory string) (*BinaryUpdateInfo, error) {
+	// Pull all available versions from library
+	validVersionsInLibrary, _, err := sortedVersionsInLibrary(binary, baseUpdateDirectory)
+	if err != nil {
+		return nil, fmt.Errorf("could not get sorted versions in library for %s: %w", binary, err)
+	}
+
+	// No valid versions in the library
+	if len(validVersionsInLibrary) < 1 {
+		return nil, errors.New("no versions in library")
+	}
+
+	// Versions are sorted in ascending order -- return the last one
+	mostRecentVersionInLibraryRaw := validVersionsInLibrary[len(validVersionsInLibrary)-1]
+	versionDir := filepath.Join(updatesDirectory(binary, baseUpdateDirectory), mostRecentVersionInLibraryRaw)
+	return &BinaryUpdateInfo{
+		Path:    executableLocation(versionDir, binary),
+		Version: mostRecentVersionInLibraryRaw,
+	}, nil
+}

pkg/autoupdate/tuf/library_lookup_test.go

@@ -0,0 +1,163 @@
+package tuf
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/go-kit/kit/log"
+	tufci "github.com/kolide/launcher/pkg/autoupdate/tuf/ci"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCheckOutLatest_withTufRepository(t *testing.T) {
+	t.Parallel()
+
+	for _, binary := range binaries {
+		binary := binary
+		t.Run(string(binary), func(t *testing.T) {
+			t.Parallel()
+
+			// Set up an update library
+			rootDir := t.TempDir()
+			updateDir := defaultLibraryDirectory(rootDir)
+
+			// Set up a local TUF repo
+			tufDir := LocalTufDirectory(rootDir)
+			require.NoError(t, os.MkdirAll(tufDir, 488))
+			testReleaseVersion := "1.0.30"
+			expectedTargetName := fmt.Sprintf("%s-%s.tar.gz", binary, testReleaseVersion)
+			tufci.SeedLocalTufRepo(t, testReleaseVersion, rootDir)
+
+			// Create a corresponding downloaded target
+			executablePath, executableVersion := pathToTargetVersionExecutable(binary, expectedTargetName, updateDir)
+			require.NoError(t, os.MkdirAll(filepath.Dir(executablePath), 0755))
+			tufci.CopyBinary(t, executablePath)
+			require.NoError(t, os.Chmod(executablePath, 0755))
+
+			// Make a more recent version that we should ignore since it isn't the release version
+			tooRecentTarget := fmt.Sprintf("%s-2.1.1.tar.gz", binary)
+			tooRecentPath, _ := pathToTargetVersionExecutable(binary, tooRecentTarget, updateDir)
+			require.NoError(t, os.MkdirAll(filepath.Dir(tooRecentPath), 0755))
+			tufci.CopyBinary(t, tooRecentPath)
+			require.NoError(t, os.Chmod(tooRecentPath, 0755))
+
+			// Check it
+			latest, err := CheckOutLatest(binary, rootDir, "", "stable", log.NewNopLogger())
+			require.NoError(t, err, "unexpected error on checking out latest")
+			require.Equal(t, executablePath, latest.Path)
+			require.Equal(t, executableVersion, latest.Version)
+		})
+	}
+}
+
+func TestCheckOutLatest_withoutTufRepository(t *testing.T) {
+	t.Parallel()
+
+	for _, binary := range binaries {
+		binary := binary
+		t.Run(string(binary), func(t *testing.T) {
+			t.Parallel()
+
+			// Set up an update library, but no TUF repo
+			rootDir := t.TempDir()
+			updateDir := defaultLibraryDirectory(rootDir)
+			target := fmt.Sprintf("%s-1.1.1.tar.gz", binary)
+			executablePath, executableVersion := pathToTargetVersionExecutable(binary, target, updateDir)
+			require.NoError(t, os.MkdirAll(filepath.Dir(executablePath), 0755))
+			tufci.CopyBinary(t, executablePath)
+			require.NoError(t, os.Chmod(executablePath, 0755))
+			_, err := os.Stat(executablePath)
+			require.NoError(t, err, "did not make test binary")
+
+			// Check it
+			latest, err := CheckOutLatest(binary, rootDir, "", "stable", log.NewNopLogger())
+			require.NoError(t, err, "unexpected error on checking out latest")
+			require.Equal(t, executablePath, latest.Path)
+			require.Equal(t, executableVersion, latest.Version)
+		})
+	}
+}
+
+func Test_mostRecentVersion(t *testing.T) {
+	t.Parallel()
+
+	for _, binary := range binaries {
+		binary := binary
+		t.Run(string(binary), func(t *testing.T) {
+			t.Parallel()
+
+			// Create update directories
+			testBaseDir := t.TempDir()
+
+			// Now, create a version in the update library
+			firstVersionTarget := fmt.Sprintf("%s-2.2.3.tar.gz", binary)
+			firstVersionPath, _ := pathToTargetVersionExecutable(binary, firstVersionTarget, testBaseDir)
+			require.NoError(t, os.MkdirAll(filepath.Dir(firstVersionPath), 0755))
+			tufci.CopyBinary(t, firstVersionPath)
+			require.NoError(t, os.Chmod(firstVersionPath, 0755))
+
+			// Create an even newer version in the update library
+			secondVersionTarget := fmt.Sprintf("%s-2.5.3.tar.gz", binary)
+			secondVersionPath, secondVersion := pathToTargetVersionExecutable(binary, secondVersionTarget, testBaseDir)
+			require.NoError(t, os.MkdirAll(filepath.Dir(secondVersionPath), 0755))
+			tufci.CopyBinary(t, secondVersionPath)
+			require.NoError(t, os.Chmod(secondVersionPath, 0755))
+
+			latest, err := mostRecentVersion(binary, testBaseDir)
+			require.NoError(t, err, "did not expect error getting most recent version")
+			require.Equal(t, secondVersionPath, latest.Path)
+			require.Equal(t, secondVersion, latest.Version)
+		})
+	}
+}
+
+func Test_mostRecentVersion_DoesNotReturnInvalidExecutables(t *testing.T) {
+	t.Parallel()
+
+	for _, binary := range binaries {
+		binary := binary
+		t.Run(string(binary), func(t *testing.T) {
+			t.Parallel()
+
+			// Create update directories
+			testBaseDir := t.TempDir()
+
+			// Now, create a version in the update library
+			firstVersionTarget := fmt.Sprintf("%s-2.2.3.tar.gz", binary)
+			firstVersionPath, firstVersion := pathToTargetVersionExecutable(binary, firstVersionTarget, testBaseDir)
+			require.NoError(t, os.MkdirAll(filepath.Dir(firstVersionPath), 0755))
+			tufci.CopyBinary(t, firstVersionPath)
+			require.NoError(t, os.Chmod(firstVersionPath, 0755))
+
+			// Create an even newer, but also corrupt, version in the update library
+			secondVersionTarget := fmt.Sprintf("%s-2.1.12.tar.gz", binary)
+			secondVersionPath, _ := pathToTargetVersionExecutable(binary, secondVersionTarget, testBaseDir)
+			require.NoError(t, os.MkdirAll(filepath.Dir(secondVersionPath), 0755))
+			os.WriteFile(secondVersionPath, []byte{}, 0755)
+
+			latest, err := mostRecentVersion(binary, testBaseDir)
+			require.NoError(t, err, "did not expect error getting most recent version")
+			require.Equal(t, firstVersionPath, latest.Path)
+			require.Equal(t, firstVersion, latest.Version)
+		})
+	}
+}
+
+func Test_mostRecentVersion_ReturnsErrorOnNoUpdatesDownloaded(t *testing.T) {
+	t.Parallel()
+
+	for _, binary := range binaries {
+		binary := binary
+		t.Run(string(binary), func(t *testing.T) {
+			t.Parallel()
+
+			// Create update directories
+			testBaseDir := t.TempDir()
+
+			_, err := mostRecentVersion(binary, testBaseDir)
+			require.Error(t, err, "should have returned error when there are no available updates")
+		})
+	}
+}