Added migration to update software_titles.

server/datastore/mysql/migrations/tables/20240607133721_ReconcileSoftwareTitles.go

@@ -0,0 +1,53 @@
+package tables
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+func init() {
+	MigrationClient.AddMigration(Up_20240607133721, Down_20240607133721)
+}
+
+func Up_20240607133721(tx *sql.Tx) error {
+
+	// For users that are not running vulnerabilities job, we need to ensure that software_titles are up-to-date
+
+	upsertTitlesStmt := `
+INSERT INTO software_titles (name, source, browser)
+SELECT DISTINCT
+	name,
+	source,
+	browser
+FROM
+	software s
+WHERE
+	NOT EXISTS (SELECT 1 FROM software_titles st WHERE (s.name, s.source, s.browser) = (st.name, st.source, st.browser))
+ON DUPLICATE KEY UPDATE software_titles.id = software_titles.id`
+
+	_, err := tx.Exec(upsertTitlesStmt)
+	if err != nil {
+		return fmt.Errorf("failed to upsert software titles: %w", err)
+	}
+
+	// update title ids for software table entries
+	updateSoftwareStmt := `
+UPDATE
+	software s,
+	software_titles st
+SET
+	s.title_id = st.id
+WHERE
+	(s.name, s.source, s.browser) = (st.name, st.source, st.browser)
+	AND (s.title_id IS NULL OR s.title_id != st.id)`
+
+	_, err = tx.Exec(updateSoftwareStmt)
+	if err != nil {
+		return fmt.Errorf("failed to update software title_id: %w", err)
+	}
+	return nil
+}
+
+func Down_20240607133721(tx *sql.Tx) error {
+	return nil
+}

server/datastore/mysql/migrations/tables/20240607133721_ReconcileSoftwareTitles_test.go

@@ -0,0 +1,53 @@
+package tables
+
+import (
+	"context"
+	"github.com/fleetdm/fleet/v4/server/fleet"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestUp_20240607133721(t *testing.T) {
+	db := applyUpToPrev(t)
+
+	// Insert data into software_titles
+	title1 := execNoErrLastID(t, db, "INSERT INTO software_titles (name, source, browser) VALUES (?, ?, ?)", "sw1", "src1", "")
+
+	// Insert software
+	const insertStmt = `INSERT INTO software
+		(name, version, source, browser, checksum, title_id)
+	VALUES
+		(?, ?, ?, ?, ?, ?)`
+
+	execNoErr(t, db, insertStmt, "sw1", "1.0", "src1", "", "1", title1)
+	execNoErr(t, db, insertStmt, "sw1", "1.0.1", "src1", "", "1a", nil)
+	execNoErr(t, db, insertStmt, "sw2", "2.0", "src2", "", "2", nil)
+	execNoErr(t, db, insertStmt, "sw3", "3.0", "src3", "browser3", "3", nil)
+
+	applyNext(t, db)
+
+	var softwareTitles []fleet.SoftwareTitle
+	require.NoError(t, db.SelectContext(context.Background(), &softwareTitles, `SELECT * FROM software_titles`))
+	require.Len(t, softwareTitles, 3)
+
+	var software []fleet.Software
+	require.NoError(t, db.SelectContext(context.Background(), &software, `SELECT id, name, source, browser, title_id FROM software`))
+	require.Len(t, software, 4)
+
+	for _, sw := range software {
+		require.NotNil(t, sw.TitleID)
+		var found bool
+		for _, title := range softwareTitles {
+			if *sw.TitleID == title.ID {
+				assert.Equal(t, sw.Name, title.Name)
+				assert.Equal(t, sw.Source, title.Source)
+				assert.Equal(t, sw.Browser, title.Browser)
+				found = true
+				break
+			}
+		}
+		assert.True(t, found)
+	}
+
+}

server/datastore/mysql/software.go

@@ -443,7 +443,7 @@ func (ds *Datastore) getExistingSoftware(
 		for checksum := range newSoftware {
 			sw := incomingByChecksum[checksum]
 			args = append(args, sw.Name, sw.Source, sw.Browser)
-			// Map software titles to software checksums
+			// Map software title identifier to software checksums so that we can map checksums to actual titles later.
 			uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(sw.Name, sw.Source, sw.Browser)] = checksum
 		}
 		var existingSoftwareTitlesForNewSoftware []fleet.SoftwareTitle
@@ -452,8 +452,6 @@ func (ds *Datastore) getExistingSoftware(
 		}
 
 		// Map software titles to software checksums.
-		// First, we map the existing software titles to the software checksums.
-
 		for _, title := range existingSoftwareTitlesForNewSoftware {
 			checksum, ok := uniqueTitleStrToChecksum[UniqueSoftwareTitleStr(title.Name, title.Source, title.Browser)]
 			if ok {
@@ -545,8 +543,6 @@ func (ds *Datastore) insertNewInstalledHostSoftwareDB(
 
 	// For software items that don't already exist in the software table, we insert them.
 	if len(softwareChecksums) > 0 {
-		// Map software titles to software checksums
-
 		keys := make([]string, 0, len(softwareChecksums))
 		for checksum := range softwareChecksums {
 			keys = append(keys, checksum)
@@ -594,11 +590,6 @@ func (ds *Datastore) insertNewInstalledHostSoftwareDB(
 				return nil, ctxerr.Wrap(ctx, err, "insert software")
 			}
 
-			// Check which titles, if any need to be inserted.
-			// We will do a SELECT and then an INSERT into software_titles for each batch of software items.
-			// Although doing INSERT IGNORE may seem more efficient since it is only one query, it may actually be slower,
-			// and it will lock rows, potentially causing more deadlocks than desired.
-
 			// Insert into software_titles
 			totalTitlesToProcess := len(newTitlesNeeded)
 			if totalTitlesToProcess > 0 {
@@ -616,7 +607,7 @@ func (ds *Datastore) insertNewInstalledHostSoftwareDB(
 					return nil, ctxerr.Wrap(ctx, err, "insert software_titles")
 				}
 
-				// update title ids for software table entries
+				// update new title ids for new software table entries
 				updateSoftwareStmt := `
 				UPDATE
 					software s,