feat: Reduce ipsw diff memory usage

cmd/ipsw/cmd/diff.go

@@ -47,6 +47,7 @@ func init() {
 	diffCmd.Flags().Bool("strs", false, "Diff MachO cstrings")
 	diffCmd.Flags().Bool("starts", false, "Diff MachO function starts")
 	diffCmd.Flags().Bool("ent", false, "Diff MachO entitlements")
+	diffCmd.Flags().Bool("low-memory", false, "Use disk caching to reduce RAM usage")
 	diffCmd.Flags().StringSlice("allow-list", []string{}, "Filter MachO sections to diff (e.g. __TEXT.__text)")
 	diffCmd.Flags().StringSlice("block-list", []string{}, "Remove MachO sections to diff (e.g. __TEXT.__info_plist)")
 	diffCmd.Flags().StringP("signatures", "s", "", "Path to symbolicator signatures folder")
@@ -66,6 +67,7 @@ func init() {
 	viper.BindPFlag("diff.strs", diffCmd.Flags().Lookup("strs"))
 	viper.BindPFlag("diff.starts", diffCmd.Flags().Lookup("starts"))
 	viper.BindPFlag("diff.ent", diffCmd.Flags().Lookup("ent"))
+	viper.BindPFlag("diff.low-memory", diffCmd.Flags().Lookup("low-memory"))
 	viper.BindPFlag("diff.files", diffCmd.Flags().Lookup("files"))
 	viper.BindPFlag("diff.allow-list", diffCmd.Flags().Lookup("allow-list"))
 	viper.BindPFlag("diff.block-list", diffCmd.Flags().Lookup("block-list"))
@@ -114,6 +116,7 @@ var diffCmd = &cobra.Command{
 			Signatures:   viper.GetString("diff.signatures"),
 			Output:       viper.GetString("diff.output"),
 			Verbose:      Verbose,
+			LowMemory:    viper.GetBool("diff.low-memory"),
 		})
 		if err := d.Diff(); err != nil {
 			return err

internal/commands/dsc/diff.go

@@ -1,10 +1,12 @@
 package dsc
 
 import (
+	"sort"
+
 	"github.com/apex/log"
 
-	"github.com/blacktop/go-macho"
 	mcmd "github.com/blacktop/ipsw/internal/commands/macho"
+	"github.com/blacktop/ipsw/internal/utils"
 	"github.com/blacktop/ipsw/pkg/dyld"
 )
 
@@ -14,60 +16,84 @@ func Diff(f1 *dyld.File, f2 *dyld.File, conf *mcmd.DiffConfig) (*mcmd.MachoDiff,
 		Updated: make(map[string]string),
 	}
 
-	/* PREVIOUS DSC */
+	// Build name->image lookup tables (cheap) so we can diff without
+	// materializing DiffInfo for every image at once.
+	prev := make(map[string]*dyld.CacheImage, len(f1.Images))
+	var prevKeys []string
+	for _, img := range f1.Images {
+		prev[img.Name] = img
+		prevKeys = append(prevKeys, img.Name)
+	}
+	next := make(map[string]*dyld.CacheImage, len(f2.Images))
+	var nextKeys []string
+	for _, img := range f2.Images {
+		next[img.Name] = img
+		nextKeys = append(nextKeys, img.Name)
+	}
 
-	prev := make(map[string]*mcmd.DiffInfo)
+	sort.Strings(prevKeys)
+	sort.Strings(nextKeys)
 
-	for _, img := range f1.Images {
-		m, err := img.GetMacho()
+	diff.New = utils.Difference(nextKeys, prevKeys)
+	diff.Removed = utils.Difference(prevKeys, nextKeys)
+	// Keep output stable for callers
+	sort.Strings(diff.New)
+	sort.Strings(diff.Removed)
+
+	for _, name := range nextKeys {
+		img1, ok := prev[name]
+		if !ok {
+			continue
+		}
+		img2 := next[name]
+
+		m1, err := img1.GetMacho()
 		if err != nil {
-			// return nil, fmt.Errorf("failed to create MachO for image %s: %v", img.Name, err)
-			log.Errorf("failed to parse MachO for image %s: %v", img.Name, err)
+			log.Errorf("failed to parse MachO for image %s: %v", img1.Name, err)
 			continue
 		}
-		// add private symbols to the macho
-		if err := img.ParseLocalSymbols(false); err == nil {
-			for _, lsym := range img.LocalSymbols {
-				m.Symtab.Syms = append(m.Symtab.Syms, macho.Symbol{
-					Name:  lsym.Name,
-					Value: lsym.Value,
-					Type:  lsym.Type,
-					Desc:  lsym.Desc,
-					Sect:  lsym.Sect,
-				})
+
+		m2, err := img2.GetMacho()
+		if err != nil {
+			log.Errorf("failed to parse MachO for image %s: %v", img2.Name, err)
+			if m1 != nil {
+				m1.Close()
 			}
+			continue
 		}
-		prev[img.Name] = mcmd.GenerateDiffInfo(m, conf)
-	}
 
-	/* NEXT DSC */
-
-	next := make(map[string]*mcmd.DiffInfo)
-
-	for _, img := range f2.Images {
-		m, err := img.GetMacho()
-		if err != nil {
-			// return nil, fmt.Errorf("failed to create MachO for image %s: %v", img.Name, err)
-			log.Errorf("failed to parse MachO for image %s: %v", img.Name, err)
+		info1 := mcmd.GenerateDiffInfo(m1, conf)
+		info2 := mcmd.GenerateDiffInfo(m2, conf)
+		if info2.Equal(*info1) {
+			if m1 != nil {
+				m1.Close()
+			}
+			if m2 != nil {
+				m2.Close()
+			}
 			continue
 		}
-		// add private symbols to the macho
-		if err := img.ParseLocalSymbols(false); err == nil {
-			for _, lsym := range img.LocalSymbols {
-				m.Symtab.Syms = append(m.Symtab.Syms, macho.Symbol{
-					Name:  lsym.Name,
-					Value: lsym.Value,
-					Type:  lsym.Type,
-					Desc:  lsym.Desc,
-					Sect:  lsym.Sect,
-				})
+
+		out, err := mcmd.FormatUpdatedDiff(info1, info2, conf)
+		if err != nil {
+			if m1 != nil {
+				m1.Close()
+			}
+			if m2 != nil {
+				m2.Close()
 			}
+			return nil, err
+		}
+		if out != "" {
+			diff.Updated[name] = out
 		}
-		next[img.Name] = mcmd.GenerateDiffInfo(m, conf)
-	}
 
-	if err := diff.Generate(prev, next, conf); err != nil {
-		return nil, err
+		if m1 != nil {
+			m1.Close()
+		}
+		if m2 != nil {
+			m2.Close()
+		}
 	}
 
 	return diff, nil