RFC: upstream recent fast-dedup and output updates from fork

.gitignore

@@ -13,3 +13,4 @@ dedup.x86_64
 dedup.universal
 test/test-data
 test/dedup_check
+.cache/

Makefile

@@ -40,7 +40,13 @@ OBJECTS = \
     map.o \
     progress.o \
     queue.o \
+    seen_set.o \
     utils.o \
+    signature.o \
+    sig_table.o \
+    runtime_caps.o \
+    runtime_dispatch.o \
+    output_format.o \
 
 .PHONY: \
     all install uninstall clean check dist distcheck \
@@ -50,19 +56,17 @@ OBJECTS = \
     clean-coverage report-coverage \
     universal-dedup universal-dist \
 	compiledb tidy \
-    list
+    list mkdirs
 
 all: dedup
 
-%.o: %.c %.h
-	rm -f $(basename $<).gcda $(basename $<).gcno
-	$(CC) $(CFLAGS) -v -c -o $@ $<
+dedup.o: CFLAGS += -I/opt/homebrew/include
 
-dedup.arm: CFLAGS += -target arm64-apple-macos11
-dedup.x86_64: CFLAGS += -target x86_64-apple-macos11
+dedup.arm: CFLAGS += -target arm64-apple-macos11 -I/opt/homebrew/include
+dedup.x86_64: CFLAGS += -target x86_64-apple-macos11 -I/opt/homebrew/include
 
 dedup dedup.arm dedup.x86_64: $(OBJECTS)
-	$(CC) $(CFLAGS) -o $@ $^
+	$(CC) $(CFLAGS) -o $@ $^ -L/opt/homebrew/lib -lxxhash
 	mv $@ $@.unsigned
 	codesign -s - -v -f $(ENTITLEMENT_FLAGS) $@.unsigned
 	mv $@.unsigned $@
@@ -118,11 +122,11 @@ report-coverage:
 
 PREFIX ?= /usr/local
 
-install: dedup
+install: dedup mkdirs
 	install dedup $(PREFIX)/bin
 	install dedup.1 $(PREFIX)/share/man/man1
 
-build/dist:
+build/dist mkdirs:
 	mkdir -p $(PREFIX)/bin
 	mkdir -p $(PREFIX)/share/man/man1
 

README.md

@@ -65,6 +65,37 @@ permissions, there should be little issue. The created files are also not
 copy-on-write and will share any modifications made. These options should only
 be used if the consequences of each choice are understood.
 
+## FAST MODE
+
+**dedup-fast** provides a high-performance alternative to the traditional
+two-phase deduplication approach. Instead of computing full SHA256 hashes for
+all files and then deduplicating in a second pass, **dedup-fast** uses a
+signature-based approach with immediate cloning.
+
+### Signature-Based Deduplication
+
+**dedup-fast** computes lightweight signatures for each file using:
+
+- **Strategic sampling**: Reads 4-byte samples at 0%, 33%, 66%, and 100% positions
+- **Quick hash**: xxHash64 of the first 4KB (or entire file if smaller)
+- **NEON acceleration**: Uses ARM NEON instructions for vectorized comparisons when available
+
+Files with matching signatures are considered duplicates and cloned immediately
+during the scan phase, eliminating the need for a separate deduplication pass.
+
+### Performance Benefits
+
+- **Single-pass operation**: No separate scan and dedup phases
+- **Reduced I/O**: Avoids full file reads for SHA256 computation
+- **Memory efficient**: Uses hash table instead of complex red-black trees
+- **Immediate cloning**: Duplicates are resolved as soon as they're found
+
+### Limitations
+
+- **Higher false positive rate**: Signature collisions may occur (rare)
+- **No SHA256 verification**: Relies on signature accuracy
+- **Memory usage**: Hash table grows with unique file count
+
 # OPTIONS
 
 The following options are available: