Merge branch 'v0.37' into janez/migration-mainnet-add-keys-to-core-co…

…ntracts-2

Makefile

@@ -540,6 +540,14 @@ docker-native-build-access-corrupt:
 		-t "$(CONTAINER_REGISTRY)/access-corrupted:$(IMAGE_TAG)" .
 	./insecure/cmd/mods_restore.sh
 
+# build a binary to run on bare metal without using docker.
+# binary is written to file ./bin/app
+.PHONY: docker-native-build-access-binary
+docker-native-build-access-binary: docker-native-build-access
+	docker create --name extract "$(CONTAINER_REGISTRY)/access:latest"
+	docker cp extract:/bin/app ./flow_access_node
+	docker rm extract
+
 .PHONY: docker-native-build-observer
 docker-native-build-observer:
 	docker build -f cmd/Dockerfile --build-arg TARGET=./cmd/observer --build-arg COMMIT=$(COMMIT) --build-arg VERSION=$(IMAGE_TAG) --build-arg GOARCH=$(GOARCH) --build-arg CGO_FLAG=$(CRYPTO_FLAG) --target production \

README.md

@@ -109,6 +109,17 @@ Build a Docker image for a particular node role (replace `$ROLE` with `collectio
 make docker-native-build-$ROLE
 ```
 
+#### Building a binary for the access node
+
+Build the binary for an access node that can be run directly on the machine without using Docker.
+
+```bash
+make docker-native-build-access-binary
+```
+_this builds a binary for Linux/x86_64 machine_.
+
+The make command will generate a binary called `flow_access_node`
+
 ### Importing the module
 
 When importing the `github.com/onflow/flow-go` module in your Go project, testing or building your project may require setting extra Go flags because the module requires [cgo](https://pkg.go.dev/cmd/cgo). In particular, `CGO_ENABLED` must be set to `1` if `cgo` isn't enabled by default. This constraint comes from the underlying cryptography library. Refer to the [crypto repository build](https://github.com/onflow/crypto?tab=readme-ov-file#build) for more details.

cmd/bootstrap/cmd/pull.go

@@ -1,21 +1,26 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/semaphore"
 
 	"github.com/onflow/flow-go/cmd"
 	"github.com/onflow/flow-go/cmd/bootstrap/gcs"
+	"github.com/onflow/flow-go/cmd/bootstrap/utils"
 )
 
 var (
-	flagNetwork    string
-	flagBucketName string
+	flagNetwork     string
+	flagBucketName  string
+	flagConcurrency int64
 )
 
 // pullCmd represents a command to pull parnter node details from the google
@@ -37,6 +42,7 @@ func addPullCmdFlags() {
 	cmd.MarkFlagRequired(pullCmd, "network")
 
 	pullCmd.Flags().StringVar(&flagBucketName, "bucket", "flow-genesis-bootstrap", "google bucket name")
+	pullCmd.Flags().Int64Var(&flagConcurrency, "concurrency", 2, "concurrency limit")
 }
 
 // pull partner node info from google bucket
@@ -62,15 +68,35 @@ func pull(cmd *cobra.Command, args []string) {
 	}
 	log.Info().Msgf("found %d files in google bucket", len(files))
 
+	sem := semaphore.NewWeighted(flagConcurrency)
+	wait := sync.WaitGroup{}
 	for _, file := range files {
-		if strings.Contains(file, "node-info.pub") {
-			fullOutpath := filepath.Join(flagOutdir, file)
-			log.Printf("downloading %s", file)
-
-			err = bucket.DownloadFile(ctx, client, fullOutpath, file)
-			if err != nil {
-				log.Error().Msgf("error trying download google bucket file: %v", err)
+		wait.Add(1)
+		go func(file gcs.GCSFile) {
+			_ = sem.Acquire(ctx, 1)
+			defer func() {
+				sem.Release(1)
+				wait.Done()
+			}()
+
+			if strings.Contains(file.Name, "node-info.pub") {
+				fullOutpath := filepath.Join(flagOutdir, file.Name)
+
+				fmd5 := utils.CalcMd5(fullOutpath)
+				// only skip files that have an MD5 hash
+				if file.MD5 != nil && bytes.Equal(fmd5, file.MD5) {
+					log.Printf("skipping %s", file)
+					return
+				}
+
+				log.Printf("downloading %s", file)
+				err = bucket.DownloadFile(ctx, client, fullOutpath, file.Name)
+				if err != nil {
+					log.Error().Msgf("error trying download google bucket file: %v", err)
+				}
 			}
-		}
+		}(file)
 	}
+
+	wait.Wait()
 }

cmd/bootstrap/gcs/gcs.go

@@ -33,14 +33,19 @@ func (g *googleBucket) NewClient(ctx context.Context) (*storage.Client, error) {
 	return client, nil
 }
 
+type GCSFile struct {
+	Name string
+	MD5  []byte
+}
+
 // GetFiles returns a list of file names within the Google bucket
-func (g *googleBucket) GetFiles(ctx context.Context, client *storage.Client, prefix, delimiter string) ([]string, error) {
+func (g *googleBucket) GetFiles(ctx context.Context, client *storage.Client, prefix, delimiter string) ([]GCSFile, error) {
 	it := client.Bucket(g.Name).Objects(ctx, &storage.Query{
 		Prefix:    prefix,
 		Delimiter: delimiter,
 	})
 
-	var files []string
+	var files []GCSFile
 	for {
 		attrs, err := it.Next()
 		if err == iterator.Done {
@@ -50,7 +55,10 @@ func (g *googleBucket) GetFiles(ctx context.Context, client *storage.Client, pre
 			return nil, err
 		}
 
-		files = append(files, attrs.Name)
+		files = append(files, GCSFile{
+			Name: attrs.Name,
+			MD5:  attrs.MD5,
+		})
 	}
 
 	return files, nil

cmd/bootstrap/transit/cmd/flags.go

@@ -9,6 +9,7 @@ var (
 	flagAccessAddress string
 	flagNodeRole      string
 	flagTimeout       time.Duration
+	flagConcurrency   int64
 
 	flagWrapID   string // wrap ID
 	flagVoteFile string

cmd/bootstrap/transit/cmd/pull.go

@@ -1,16 +1,20 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/fs"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/semaphore"
 
 	"github.com/onflow/flow-go/cmd/bootstrap/gcs"
+	"github.com/onflow/flow-go/cmd/bootstrap/utils"
 	model "github.com/onflow/flow-go/model/bootstrap"
 	"github.com/onflow/flow-go/model/flow"
 )
@@ -32,6 +36,7 @@ func addPullCmdFlags() {
 	pullCmd.Flags().StringVarP(&flagToken, "token", "t", "", "token provided by the Flow team to access the Transit server")
 	pullCmd.Flags().StringVarP(&flagNodeRole, "role", "r", "", `node role (can be "collection", "consensus", "execution", "verification" or "access")`)
 	pullCmd.Flags().DurationVar(&flagTimeout, "timeout", time.Second*300, `timeout for pull`)
+	pullCmd.Flags().Int64Var(&flagConcurrency, "concurrency", 2, `concurrency limit for pull`)
 
 	_ = pullCmd.MarkFlagRequired("token")
 	_ = pullCmd.MarkFlagRequired("role")
@@ -78,17 +83,34 @@ func pull(cmd *cobra.Command, args []string) {
 	}
 	log.Info().Msgf("found %d files in Google Bucket", len(files))
 
-	// download found files
+	sem := semaphore.NewWeighted(flagConcurrency)
+	wait := sync.WaitGroup{}
 	for _, file := range files {
-		fullOutpath := filepath.Join(flagBootDir, "public-root-information", filepath.Base(file))
-
-		log.Info().Str("source", file).Str("dest", fullOutpath).Msgf("downloading file from transit servers")
-		err = bucket.DownloadFile(ctx, client, fullOutpath, file)
-		if err != nil {
-			log.Fatal().Err(err).Msgf("could not download google bucket file")
-		}
+		wait.Add(1)
+		go func(file gcs.GCSFile) {
+			_ = sem.Acquire(ctx, 1)
+			defer func() {
+				sem.Release(1)
+				wait.Done()
+			}()
+
+			fullOutpath := filepath.Join(flagBootDir, "public-root-information", filepath.Base(file.Name))
+			fmd5 := utils.CalcMd5(fullOutpath)
+			// only skip files that have an MD5 hash
+			if file.MD5 != nil && bytes.Equal(fmd5, file.MD5) {
+				log.Info().Str("source", file.Name).Str("dest", fullOutpath).Msgf("skipping existing file from transit servers")
+				return
+			}
+			log.Info().Str("source", file.Name).Str("dest", fullOutpath).Msgf("downloading file from transit servers")
+			err = bucket.DownloadFile(ctx, client, fullOutpath, file.Name)
+			if err != nil {
+				log.Fatal().Err(err).Msgf("could not download google bucket file")
+			}
+		}(file)
 	}
 
+	wait.Wait()
+
 	// download any extra files specific to node role
 	extraFiles := getAdditionalFilesToDownload(role, nodeID)
 	for _, file := range extraFiles {

cmd/bootstrap/utils/md5.go

@@ -0,0 +1,25 @@
+package utils
+
+// The google storage API only provides md5 and crc32 hence overriding the linter flag for md5
+// #nosec
+import (
+	"crypto/md5"
+	"io"
+	"os"
+)
+
+func CalcMd5(outpath string) []byte {
+	f, err := os.Open(outpath)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+
+	// #nosec
+	h := md5.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return nil
+	}
+
+	return h.Sum(nil)
+}

cmd/util/cmd/check-storage/cmd.go

@@ -2,6 +2,7 @@ package check_storage
 
 import (
 	"context"
+	"slices"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -14,7 +15,9 @@ import (
 	"github.com/onflow/flow-go/cmd/util/ledger/reporters"
 	"github.com/onflow/flow-go/cmd/util/ledger/util"
 	"github.com/onflow/flow-go/cmd/util/ledger/util/registers"
+	"github.com/onflow/flow-go/fvm/systemcontracts"
 	"github.com/onflow/flow-go/ledger"
+	"github.com/onflow/flow-go/model/flow"
 	moduleUtil "github.com/onflow/flow-go/module/util"
 )
 
@@ -23,6 +26,7 @@ var (
 	flagState           string
 	flagStateCommitment string
 	flagOutputDirectory string
+	flagChain           string
 	flagNWorker         int
 )
 
@@ -74,10 +78,22 @@ func init() {
 		10,
 		"number of workers to use",
 	)
+
+	Cmd.Flags().StringVar(
+		&flagChain,
+		"chain",
+		"",
+		"Chain name",
+	)
+	_ = Cmd.MarkFlagRequired("chain")
 }
 
 func run(*cobra.Command, []string) {
 
+	chainID := flow.ChainID(flagChain)
+	// Validate chain ID
+	_ = chainID.Chain()
+
 	if flagPayloads == "" && flagState == "" {
 		log.Fatal().Msg("Either --payloads or --state must be provided")
 	} else if flagPayloads != "" && flagState != "" {
@@ -87,6 +103,14 @@ func run(*cobra.Command, []string) {
 		log.Fatal().Msg("--state-commitment must be provided when --state is provided")
 	}
 
+	// For now, skip EVM storage account since a different decoder is needed for decoding EVM registers.
+
+	systemContracts := systemcontracts.SystemContractsForChain(chainID)
+
+	acctsToSkip := []string{
+		flow.AddressToRegisterOwner(systemContracts.EVMStorage.Address),
+	}
+
 	// Create report in JSONL format
 	rw := reporters.NewReportFileWriterFactoryWithFormat(flagOutputDirectory, log.Logger, reporters.ReportFormatJSONL).
 		ReportWriter(ReporterName)
@@ -138,7 +162,7 @@ func run(*cobra.Command, []string) {
 		len(payloads),
 	)
 
-	failedAccountAddresses, err := checkStorageHealth(registersByAccount, flagNWorker, rw)
+	failedAccountAddresses, err := checkStorageHealth(registersByAccount, flagNWorker, rw, acctsToSkip)
 	if err != nil {
 		log.Fatal().Err(err).Msg("failed to check storage health")
 	}
@@ -165,6 +189,7 @@ func checkStorageHealth(
 	registersByAccount *registers.ByAccount,
 	nWorkers int,
 	rw reporters.ReportWriter,
+	acctsToSkip []string,
 ) (failedAccountAddresses []string, err error) {
 
 	accountCount := registersByAccount.AccountCount()
@@ -185,6 +210,12 @@ func checkStorageHealth(
 		// Skip goroutine to avoid overhead
 		err = registersByAccount.ForEachAccount(
 			func(accountRegisters *registers.AccountRegisters) error {
+				defer logAccount(1)
+
+				if slices.Contains(acctsToSkip, accountRegisters.Owner()) {
+					return nil
+				}
+
 				accountStorageIssues := checkAccountStorageHealth(accountRegisters, nWorkers)
 
 				if len(accountStorageIssues) > 0 {
@@ -195,8 +226,6 @@ func checkStorageHealth(
 					}
 				}
 
-				logAccount(1)
-
 				return nil
 			})
 

cmd/util/ledger/migrations/account_storage_migration.go

@@ -52,25 +52,15 @@ func NewAccountStorageMigration(
 			log.Err(err).Msg("storage health check failed")
 		}
 
-		// Finalize the transaction
-		result, err := migrationRuntime.TransactionState.FinalizeMainTransaction()
-		if err != nil {
-			return fmt.Errorf("failed to finalize main transaction: %w", err)
-		}
+		// Commit/finalize the transaction
 
-		// Merge the changes into the registers
 		expectedAddresses := map[flow.Address]struct{}{
 			flow.Address(address): {},
 		}
 
-		err = registers.ApplyChanges(
-			registersByAccount,
-			result.WriteSet,
-			expectedAddresses,
-			log,
-		)
+		err = migrationRuntime.Commit(expectedAddresses, log)
 		if err != nil {
-			return fmt.Errorf("failed to apply register changes: %w", err)
+			return fmt.Errorf("failed to commit: %w", err)
 		}
 
 		return nil