Fix a collection of downloader, snapshot sync and torrent related issues

.gitignore

@@ -101,4 +101,11 @@ node_modules
 /config.yaml
 /config.yml
 
-vendor
+vendor/
+
+!/cmd/snapshots
+**/snapshots/
+mdbx.dat
+mdbx.lck
+/*/**/LOCK
+/*/**/nodekey

Makefile

@@ -1,8 +1,8 @@
 GO ?= go # if using docker, should not need to be installed/linked
 GOAMD64_VERSION ?= v2 # See https://go.dev/wiki/MinimumRequirements#microarchitecture-support
-GOBINREL = build/bin
-GOBIN = $(CURDIR)/$(GOBINREL)
-UNAME = $(shell uname) # Supported: Darwin, Linux
+GOBINREL := build/bin
+GOBIN := $(CURDIR)/$(GOBINREL)
+UNAME := $(shell uname) # Supported: Darwin, Linux
 DOCKER := $(shell command -v docker 2> /dev/null)
 
 GIT_COMMIT ?= $(shell git rev-list -1 HEAD)
@@ -46,7 +46,7 @@ ifeq ($(shell uname -s), Darwin)
 endif
 
 # about netgo see: https://github.com/golang/go/issues/30310#issuecomment-471669125 and https://github.com/golang/go/issues/57757
-BUILD_TAGS = nosqlite,noboltdb
+BUILD_TAGS = noboltdb
 
 ifneq ($(shell "$(CURDIR)/turbo/silkworm/silkworm_compat_check.sh"),)
 	BUILD_TAGS := $(BUILD_TAGS),nosilkworm
@@ -58,12 +58,17 @@ GOPRIVATE = github.com/erigontech/silkworm-go
 
 PACKAGE = github.com/erigontech/erigon
 
-override GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false
-override GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}"
+# Add to user provided GO_FLAGS. Insert it after a bunch of other stuff to allow overrides, and before tags to maintain BUILD_TAGS (set that instead if you want to modify it).
 
-GOBUILD = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" $(GO) build $(GO_FLAGS)
-GO_DBG_BUILD = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" $(GO) build -tags $(BUILD_TAGS),debug -gcflags=all="-N -l"  # see delve docs
-GOTEST = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)" GODEBUG=cgocheck=0 GOTRACEBACK=1 $(GO) test $(GO_FLAGS) ./...
+GO_RELEASE_FLAGS := -trimpath -buildvcs=false \
+	-ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}"
+GO_BUILD_ENV = ${CPU_ARCH} CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GOPRIVATE="$(GOPRIVATE)"
+
+# Basic release build. Pass EXTRA_BUILD_TAGS if you want to modify the tags set.
+GOBUILD = $(GO_BUILD_ENV) $(GO) build $(GO_RELEASE_FLAGS) $(GO_FLAGS) -tags $(BUILD_TAGS)
+DLV_GO_FLAGS := -gcflags='all="-N -l" -trimpath=false'
+GO_BUILD_DEBUG = $(GO_BUILD_ENV) CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" $(GO) build $(DLV_GO_FLAGS) $(GO_FLAGS) -tags $(BUILD_TAGS),debug
+GOTEST = $(GO_BUILD_ENV) GODEBUG=cgocheck=0 GOTRACEBACK=1 $(GO) test $(GO_FLAGS) ./...
 
 default: all
 
@@ -117,12 +122,14 @@ docker-compose: validate_docker_build_args setup_xdg_data_home
 
 ## dbg                                debug build allows see C stack traces, run it with GOTRACEBACK=crash. You don't need debug build for C pit for profiling. To profile C code use SETCGOTRCKEBACK=1
 dbg:
-	$(GO_DBG_BUILD) -o $(GOBIN)/ ./cmd/...
+	$(GO_BUILD_DEBUG) -o $(GOBIN)/ ./cmd/...
 
+.PHONY: %.cmd
+# Deferred (=) because $* isn't defined until the rule is executed.
+%.cmd: override OUTPUT = $(GOBIN)/$*$(CMD_BUILD_SUFFIX)
 %.cmd:
-	@# Note: $* is replaced by the command name
-	@echo "Building $*"
-	@cd ./cmd/$* && $(GOBUILD) -o $(GOBIN)/$*
+	@echo Building '$(OUTPUT)'
+	cd ./cmd/$* && $(GOBUILD) -o $(OUTPUT)
 	@echo "Run \"$(GOBIN)/$*\" to launch $*."
 
 ## geth:                              run erigon (TODO: remove?)
@@ -226,7 +233,7 @@ test-all-race: test-erigon-lib-all-race test-erigon-db-all-race test-p2p-all-rac
 	}
 
 ## test-hive						run the hive tests locally off nektos/act workflows simulator
-test-hive:	
+test-hive:
 	@if ! command -v act >/dev/null 2>&1; then \
 		echo "act command not found in PATH, please source it in PATH. If nektosact is not installed, install it by visiting https://nektosact.com/installation/index.html"; \
 	elif [ -z "$(GITHUB_TOKEN)"]; then \
@@ -285,7 +292,7 @@ hive-local:
 
 eest-hive:
 	@if [ ! -d "temp" ]; then mkdir temp; fi
-	docker build -t "test/erigon:$(SHORT_COMMIT)" . 
+	docker build -t "test/erigon:$(SHORT_COMMIT)" .
 	rm -rf "temp/eest-hive-$(SHORT_COMMIT)" && mkdir "temp/eest-hive-$(SHORT_COMMIT)"
 	cd "temp/eest-hive-$(SHORT_COMMIT)" && git clone https://github.com/erigontech/hive
 	cd "temp/eest-hive-$(SHORT_COMMIT)/hive" && \
@@ -316,7 +323,7 @@ check-kurtosis:
 kurtosis-pectra-assertoor:	check-kurtosis
 	@$(call run-kurtosis-assertoor,".github/workflows/kurtosis/pectra.io")
 
-kurtosis-regular-assertoor:	check-kurtosis 
+kurtosis-regular-assertoor:	check-kurtosis
 	@$(call run-kurtosis-assertoor,".github/workflows/kurtosis/regular-assertoor.io")
 
 kurtosis-fusaka-assertoor: check-kurtosis
@@ -347,10 +354,10 @@ lint:
 
 ## tidy:                              `go mod tidy`
 tidy:
-	@cd erigon-lib && go mod tidy
-	@cd erigon-db && go mod tidy
-	@cd p2p && go mod tidy
-	@go mod tidy
+	cd erigon-lib && go mod tidy
+	cd erigon-db && go mod tidy
+	cd p2p && go mod tidy
+	go mod tidy
 
 ## clean:                             cleans the go cache, build dir, libmdbx db dir
 clean:

cmd/downloader/main.go

@@ -25,12 +25,14 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 	"time"
 
 	"github.com/anacrolix/torrent/metainfo"
 	"github.com/c2h5oh/datasize"
+	"github.com/go-viper/mapstructure/v2"
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery"
 	"github.com/pelletier/go-toml/v2"
@@ -60,11 +62,11 @@ import (
 	"github.com/erigontech/erigon/cmd/hack/tool"
 	"github.com/erigontech/erigon/cmd/utils"
 	"github.com/erigontech/erigon/params"
+	_ "github.com/erigontech/erigon/polygon/heimdall" //hack
 	"github.com/erigontech/erigon/turbo/debug"
 	"github.com/erigontech/erigon/turbo/logging"
 
-	_ "github.com/erigontech/erigon-db/snaptype"      //hack
-	_ "github.com/erigontech/erigon/polygon/heimdall" //hack
+	_ "github.com/erigontech/erigon-db/snaptype" //hack
 )
 
 func main() {
@@ -90,17 +92,18 @@ var (
 	natSetting                     string
 	torrentVerbosity               int
 	downloadRateStr, uploadRateStr string
-	torrentDownloadSlots           int
-	staticPeersStr                 string
-	torrentPort                    int
-	torrentMaxPeers                int
-	torrentConnsPerFile            int
-	targetFile                     string
-	disableIPV6                    bool
-	disableIPV4                    bool
-	seedbox                        bool
-	dbWritemap                     bool
-	all                            bool
+	// How do I mark this deprecated with cobra?
+	torrentDownloadSlots int
+	staticPeersStr       string
+	torrentPort          int
+	torrentMaxPeers      int
+	torrentConnsPerFile  int
+	targetFile           string
+	disableIPV6          bool
+	disableIPV4          bool
+	seedbox              bool
+	dbWritemap           bool
+	all                  bool
 )
 
 func init() {
@@ -118,6 +121,7 @@ func init() {
 	rootCmd.Flags().IntVar(&torrentPort, "torrent.port", utils.TorrentPortFlag.Value, utils.TorrentPortFlag.Usage)
 	rootCmd.Flags().IntVar(&torrentMaxPeers, "torrent.maxpeers", utils.TorrentMaxPeersFlag.Value, utils.TorrentMaxPeersFlag.Usage)
 	rootCmd.Flags().IntVar(&torrentConnsPerFile, "torrent.conns.perfile", utils.TorrentConnsPerFileFlag.Value, utils.TorrentConnsPerFileFlag.Usage)
+	// Deprecated.
 	rootCmd.Flags().IntVar(&torrentDownloadSlots, "torrent.download.slots", utils.TorrentDownloadSlotsFlag.Value, utils.TorrentDownloadSlotsFlag.Usage)
 	rootCmd.Flags().StringVar(&staticPeersStr, utils.TorrentStaticPeersFlag.Name, utils.TorrentStaticPeersFlag.Value, utils.TorrentStaticPeersFlag.Usage)
 	rootCmd.Flags().BoolVar(&disableIPV6, "downloader.disable.ipv6", utils.DisableIPV6.Value, utils.DisableIPV6.Usage)
@@ -216,7 +220,7 @@ func Downloader(ctx context.Context, logger log.Logger) error {
 	if err := checkChainName(ctx, dirs, chain); err != nil {
 		return err
 	}
-	torrentLogLevel, _, err := downloadercfg.Int2LogLevel(torrentVerbosity)
+	torrentLogLevel, err := downloadercfg.Int2LogLevel(torrentVerbosity)
 	if err != nil {
 		return err
 	}
@@ -244,12 +248,26 @@ func Downloader(ctx context.Context, logger log.Logger) error {
 			return err
 		}
 	}
-	cfg, err := downloadercfg.New(ctx, dirs, version, torrentLogLevel, downloadRate, uploadRate, torrentPort, torrentConnsPerFile, torrentDownloadSlots, staticPeers, webseedsList, chain, true, dbWritemap)
+	cfg, err := downloadercfg.New(
+		ctx,
+		dirs,
+		version,
+		torrentLogLevel,
+		downloadRate,
+		uploadRate,
+		torrentPort,
+		torrentConnsPerFile,
+		staticPeers,
+		webseedsList,
+		chain,
+		dbWritemap,
+		downloadercfg.NewCfgOpts{},
+	)
 	if err != nil {
 		return err
 	}
 
-	cfg.ClientConfig.PieceHashersPerTorrent = dbg.EnvInt("DL_HASHERS", 32)
+	cfg.ClientConfig.PieceHashersPerTorrent = dbg.EnvInt("DL_HASHERS", runtime.NumCPU())
 	cfg.ClientConfig.DisableIPv6 = disableIPV6
 	cfg.ClientConfig.DisableIPv4 = disableIPV4
 
@@ -259,9 +277,12 @@ func Downloader(ctx context.Context, logger log.Logger) error {
 	}
 	downloadernat.DoNat(natif, cfg.ClientConfig, logger)
 
-	cfg.AddTorrentsFromDisk = true // always true unless using uploader - which wants control of torrent files
+	// Called manually to ensure all torrents are present before verification.
+	cfg.AddTorrentsFromDisk = false
+	manualDataVerification := verify || verifyFailfast || len(verifyFiles) > 0
+	cfg.ManualDataVerification = manualDataVerification
 
-	d, err := downloader.New(ctx, cfg, logger, log.LvlInfo, seedbox)
+	d, err := downloader.New(ctx, cfg, logger, log.LvlInfo)
 	if err != nil {
 		return err
 	}
@@ -270,15 +291,28 @@ func Downloader(ctx context.Context, logger log.Logger) error {
 
 	d.HandleTorrentClientStatus()
 
+	err = d.AddTorrentsFromDisk(ctx)
+	if err != nil {
+		return fmt.Errorf("adding torrents from disk: %w", err)
+	}
+
+	// I think we could use DisableInitialPieceVerification to get the behaviour we want here: One
+	// hash, and fail if it's in the verify files list or we have fail fast on.
+
 	if len(_verifyFiles) > 0 {
 		verifyFiles = strings.Split(_verifyFiles, ",")
 	}
-	if verify || verifyFailfast || len(verifyFiles) > 0 { // remove and create .torrent files (will re-read all snapshots)
-		if err = d.VerifyData(ctx, verifyFiles, verifyFailfast); err != nil {
+	if manualDataVerification { // remove and create .torrent files (will re-read all snapshots)
+		if err = d.VerifyData(ctx, verifyFiles); err != nil {
 			return err
 		}
 	}
 
+	// This only works if Cfg.ManualDataVerification is held by reference by the Downloader. The
+	// alternative is to pass the value through AddTorrentsFromDisk, do it per Torrent ourselves, or
+	// defer all hashing to the torrent Client in the Downloader and wait for it to complete.
+	cfg.ManualDataVerification = false
+
 	bittorrentServer, err := downloader.NewGrpcServer(d)
 	if err != nil {
 		return fmt.Errorf("new server: %w", err)
@@ -375,14 +409,19 @@ var torrentCat = &cobra.Command{
 		if err != nil {
 			return fmt.Errorf("LoadFromFile: %w, file=%s", err, fPath)
 		}
+		var ms map[string]any
+		err = mapstructure.Decode(mi, &ms)
+		if err != nil {
+			return fmt.Errorf("decoding metainfo into map: %w", err)
+		}
 		fmt.Printf("InfoHash = '%x'\n", mi.HashInfoBytes())
-		mi.InfoBytes = nil
-		bytes, err := toml.Marshal(mi)
+		delete(ms, "InfoBytes")
+		bytes, err := toml.Marshal(ms)
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s\n", string(bytes))
-		return nil
+		_, err = os.Stdout.Write(bytes)
+		return err
 	},
 }
 var torrentClean = &cobra.Command{

cmd/erigon/main.go

@@ -24,7 +24,6 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/erigontech/erigon-lib/common/datadir"
-	"github.com/erigontech/erigon-lib/common/dbg"
 	"github.com/erigontech/erigon-lib/log/v3"
 	"github.com/erigontech/erigon-lib/metrics"
 	"github.com/erigontech/erigon-lib/version"
@@ -38,15 +37,15 @@ import (
 )
 
 func main() {
-	defer func() {
-		panicResult := recover()
-		if panicResult == nil {
-			return
-		}
-
-		log.Error("catch panic", "err", panicResult, "stack", dbg.Stack())
-		os.Exit(1)
-	}()
+	//defer func() {
+	//	panicResult := recover()
+	//	if panicResult == nil {
+	//		return
+	//	}
+	//
+	//	log.Error("catch panic", "err", panicResult, "stack", dbg.Stack())
+	//	os.Exit(1)
+	//}()
 
 	app := erigonapp.MakeApp("erigon", runErigon, erigoncli.DefaultFlags)
 	if err := app.Run(os.Args); err != nil {