merge master

ethpandaops · Oct 28, 2024 · 55662fd · 55662fd
2 parents 53d228e + eeb551b
commit 55662fd
Show file tree

Hide file tree

Showing 126 changed files with 5,862 additions and 2,175 deletions.
diff --git a/.github/cannon/assert_clickhouse.sh b/.github/cannon/assert_clickhouse.sh
@@ -12,7 +12,7 @@ SEEDING_YAML="$1"
 CLICKHOUSE_HOST=${CLICKHOUSE_HOST:-"localhost"}
 CLICKHOUSE_PORT=${CLICKHOUSE_PORT:-"9000"}
 CLICKHOUSE_USER=${CLICKHOUSE_USER:-"default"}
-CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-""}
+CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD}
 CLICKHOUSE_DB=${CLICKHOUSE_DB:-"default"}
 
 # Function to execute ClickHouse query
@@ -53,6 +53,7 @@ yq e '.networks[].types[] | [.name, .assert.query, .assert.expected] | @tsv' "$S
                     break
                 else
                     echo "Assertion failed for $name. Expected: $expected, Got: $result"
+                    docker logs xatu-cannon 2>&1 | grep -i "$name " | tail -n 10 || docker logs xatu-cannon --tail 10
                     sleep 2
                     continue
                 fi

diff --git a/.github/cannon/seeding.yaml b/.github/cannon/seeding.yaml
@@ -114,8 +114,8 @@ networks:
             AND epoch_start_date_time = '2024-08-09 02:24:00'
             AND block_root = '0xff8e97fdf4033e499681ff0eb056cf5f2877a8f9b16f14e2db3956b75190b9e7'
             AND block_version = 'deneb'
-            AND block_total_bytes = 204103
-            AND block_total_bytes_compressed = 95454
+            AND block_total_bytes = 94749
+            AND block_total_bytes_compressed = 55636
             AND parent_root = '0x54264e13e97a137bed2cc1a52cf76a94bb5e385507d34ff3902d18672f4ef79e'
             AND state_root = '0x3598ed522160cb6dd0aab216f03b26d61b32d87854c3d38754897017a66004f5'
             AND proposer_index = 416934

diff --git a/.github/workflows/cannon-smoke-test.yaml b/.github/workflows/cannon-smoke-test.yaml
@@ -75,6 +75,7 @@ jobs:
     - name: Create Xatu Cannon config
       run: |
         cat <<EOF > /tmp/cannon_config.yaml
+        logging: debug
         name: cannon-smoke-test
         labels:
           ethpandaops: rocks
@@ -97,7 +98,7 @@ jobs:
           beaconBlock:
             enabled: true
           beaconBlobSidecar:
-            enabled: true
+            enabled: false
           proposerDuty:
             enabled: true
           beaconCommittee:

diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml
@@ -20,10 +20,10 @@ jobs:
           go-version: '1.22'
       - uses: actions/checkout@v3
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v4
+        uses: golangci/golangci-lint-action@v6
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: v1.56
+          version: v1.61
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir

diff --git a/.github/workflows/sentry-smoke-test.yaml b/.github/workflows/sentry-smoke-test.yaml
@@ -134,7 +134,7 @@ jobs:
             for sentry in "${all_sentries[@]}"; do
                 pretty_print "Checking $table table for $sentry..." "none"
                 while true; do
-                    data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --query "SELECT COUNT(*) FROM default.$table WHERE meta_client_name = '$sentry'" || true)
+                    data_count=$(docker exec xatu-clickhouse-01 clickhouse-client --user=default --query "SELECT COUNT(*) FROM default.$table WHERE meta_client_name = '$sentry'" || true)
                     if [[ $data_count -gt 0 ]]; then
                         pretty_print "$table has $data_count entries from $sentry" "green"
                         break

diff --git a/.goreleaser.yaml b/.goreleaser.yaml
@@ -16,7 +16,7 @@ builds:
     goarch: 
     - amd64
     ldflags:
-      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}}
+      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOOS=linux -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOARCH=amd64
     mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: linux-arm64
@@ -30,7 +30,7 @@ builds:
     goarch: 
     - arm64
     ldflags:
-      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}}
+      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOOS=linux -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOARCH=arm64
     mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: windows-amd64
@@ -44,7 +44,7 @@ builds:
     goarch:
     - amd64
     ldflags:
-      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}}
+      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOOS=windows -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOARCH=amd64
     mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: darwin-amd64
@@ -58,7 +58,7 @@ builds:
     goarch:
     - amd64
     ldflags:
-      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}}
+      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOOS=darwin -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOARCH=amd64
     mod_timestamp: "{{ .CommitTimestamp }}"
 
   - id: darwin-arm64
@@ -72,7 +72,7 @@ builds:
     goarch:
     - arm64
     ldflags:
-      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}}
+      - -s -w -X github.com/ethpandaops/xatu/pkg/proto/xatu.Release={{.Tag}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GitCommit={{.ShortCommit}} -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOOS=darwin -X github.com/ethpandaops/xatu/pkg/proto/xatu.GOARCH=arm64
     mod_timestamp: "{{ .CommitTimestamp }}"
 checksum:
   name_template: 'checksums.txt'

diff --git a/Makefile b/Makefile
@@ -1,9 +1,2 @@
 proto:
-	protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/eth/v1 --go_out=./pkg/proto/eth/v1/ pkg/proto/eth/v1/*.proto
-	protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/eth/v2 --go_out=./pkg/proto/eth/v2/ pkg/proto/eth/v2/*.proto
-	protoc --proto_path=./ --proto_path=./pkg/proto/eth/v1 --proto_path=./pkg/proto/eth/v2 --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/xatu --go-grpc_out=. --go-grpc_opt=paths=source_relative --go_out=./pkg/proto/xatu pkg/proto/xatu/*.proto
-	protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/blockprint --go_out=./pkg/proto/blockprint pkg/proto/blockprint/*.proto
-	protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/mevrelay --go_out=./pkg/proto/mevrelay pkg/proto/mevrelay/*.proto
-	protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/libp2p --go_out=./pkg/proto/libp2p pkg/proto/libp2p/*.proto
-	protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/libp2p/gossipsub --go_out=./pkg/proto/libp2p/gossipsub pkg/proto/libp2p/gossipsub/*.proto
-
+	buf generate
diff --git a/README.md b/README.md
@@ -64,6 +64,23 @@ Download the latest release from the [Releases page](https://github.com/ethpanda
 ./xatu <server|sentry|discovery|mimicry> --config your-config.yaml
 ```
 
+### Install via bash script
+
+View the install script [here](./installer.sh). You can run the following to install:
+```
+curl -sL https://raw.githubusercontent.com/ethpandaops/xatu/master/install.sh | bash
+```
+
+To change the install location of the binary you can run and change the `BASE_INSTALL_PATH` variable:
+```
+curl -sL https://raw.githubusercontent.com/ethpandaops/xatu/master/install.sh | BASE_INSTALL_PATH=~/.local/bin bash
+```
+
+To uninstall the binary you can run:
+```
+curl -sL https://raw.githubusercontent.com/ethpandaops/xatu/master/install.sh | bash -s uninstall
+```
+
 ### Docker
 
 Available as a docker image at [ethpandaops/xatu](https://hub.docker.com/r/ethpandaops/xatu/tags)

diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -0,0 +1,8 @@
+version: v2
+plugins:
+  - remote: buf.build/protocolbuffers/go:v1.34.2
+    out: .
+    opt: paths=source_relative
+  - remote: buf.build/grpc/go:v1.5.1
+    out: .
+    opt: paths=source_relative
diff --git a/buf.lock b/buf.lock
@@ -0,0 +1,9 @@
+# Generated by buf. DO NOT EDIT.
+version: v2
+deps:
+  - name: buf.build/googleapis/googleapis
+    commit: e7f8d366f5264595bcc4cd4139af9973
+    digest: b5:0cd69a689ee320ed815663d57d1bc3a1d6823224a7a717d46fee3a68197c25a6f5f932c0b0e49f8370c70c247a6635969a6a54af5345cafd51e0667298768aca
+  - name: buf.build/grpc/grpc
+    commit: 6b465f79a38c4b8fb79a16e00d5721c5
+    digest: b5:24490539afee77935a7bcfa1d9dc7e4cfbf1dfba75a89c72f2ea1ef1b753575479018e0e1ad529906f9856d48865311829df987848a2496814901b7be0e7e27c
diff --git a/buf.yaml b/buf.yaml
@@ -0,0 +1,13 @@
+version: v2
+modules:
+  - path: .
+    name: github.com/ethpandaops/xatu
+lint:
+  use:
+    - STANDARD
+breaking:
+  use:
+    - PACKAGE
+deps:
+  - buf.build/googleapis/googleapis
+  - buf.build/grpc/grpc
diff --git a/cmd/cannon.go b/cmd/cannon.go
@@ -6,7 +6,6 @@ import (
 
 	"github.com/creasty/defaults"
 	"github.com/ethpandaops/xatu/pkg/cannon"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -15,6 +14,93 @@ var (
 	cannonCfgFile string
 )
 
+type CannonOverride struct {
+	FlagHelper func(cmd *cobra.Command)
+	Setter     func(cmd *cobra.Command, overrides *cannon.Override) error
+}
+
+type CannonOverrideConfig struct {
+	FlagName     string
+	EnvName      string
+	Description  string
+	OverrideFunc func(val string, overrides *cannon.Override)
+}
+
+func createCannonOverride(config CannonOverrideConfig) CannonOverride {
+	return CannonOverride{
+		FlagHelper: func(cmd *cobra.Command) {
+			cmd.Flags().String(config.FlagName, "", config.Description+` (env: `+config.EnvName+`)`)
+		},
+		Setter: func(cmd *cobra.Command, overrides *cannon.Override) error {
+			val := ""
+
+			if cmd.Flags().Changed(config.FlagName) {
+				val = cmd.Flags().Lookup(config.FlagName).Value.String()
+			}
+
+			if os.Getenv(config.EnvName) != "" {
+				val = os.Getenv(config.EnvName)
+			}
+
+			if val == "" {
+				return nil
+			}
+
+			config.OverrideFunc(val, overrides)
+
+			return nil
+		},
+	}
+}
+
+var CannonOverrides = []CannonOverride{
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-xatu-output-authorization",
+		EnvName:     "CANNON_XATU_OUTPUT_AUTHORIZATION",
+		Description: "sets the authorization secret for all xatu outputs",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.XatuOutputAuth.Enabled = true
+			overrides.XatuOutputAuth.Value = val
+		},
+	}),
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-xatu-coordinator-authorization",
+		EnvName:     "CANNON_XATU_COORDINATOR_AUTHORIZATION",
+		Description: "sets the authorization secret for the xatu coordinator",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.XatuCoordinatorAuth.Enabled = true
+			overrides.XatuCoordinatorAuth.Value = val
+		},
+	}),
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-beacon-node-url",
+		EnvName:     "CANNON_BEACON_NODE_URL",
+		Description: "sets the beacon node url",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.BeaconNodeURL.Enabled = true
+			overrides.BeaconNodeURL.Value = val
+		},
+	}),
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-beacon-node-authorization-header",
+		EnvName:     "CANNON_BEACON_NODE_AUTHORIZATION_HEADER",
+		Description: "sets the beacon node authorization header",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.BeaconNodeAuthorizationHeader.Enabled = true
+			overrides.BeaconNodeAuthorizationHeader.Value = val
+		},
+	}),
+	createCannonOverride(CannonOverrideConfig{
+		FlagName:    "cannon-network-name",
+		EnvName:     "CANNON_NETWORK_NAME",
+		Description: "sets the network name",
+		OverrideFunc: func(val string, overrides *cannon.Override) {
+			overrides.NetworkName.Enabled = true
+			overrides.NetworkName.Value = val
+		},
+	}),
+}
+
 // cannonCmd represents the cannon command
 var cannonCmd = &cobra.Command{
 	Use:   "cannon",
@@ -24,23 +110,24 @@ var cannonCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		initCommon()
 
-		log.WithField("location", cannonCfgFile).Info("Loading config")
-
 		config, err := loadcannonConfigFromFile(cannonCfgFile)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		log.Info("Config loaded")
+		log = getLogger(config.LoggingLevel, "")
 
-		logLevel, err := logrus.ParseLevel(config.LoggingLevel)
-		if err != nil {
-			log.WithField("logLevel", config.LoggingLevel).Fatal("invalid logging level")
-		}
+		log.WithField("location", cannonCfgFile).Info("Loaded config")
+
+		overrides := &cannon.Override{}
 
-		log.SetLevel(logLevel)
+		for _, override := range CannonOverrides {
+			if errr := override.Setter(cmd, overrides); errr != nil {
+				log.Fatal(errr)
+			}
+		}
 
-		cannon, err := cannon.New(cmd.Context(), log, config)
+		cannon, err := cannon.New(cmd.Context(), log, config, overrides)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -57,6 +144,10 @@ func init() {
 	rootCmd.AddCommand(cannonCmd)
 
 	cannonCmd.Flags().StringVar(&cannonCfgFile, "config", "cannon.yaml", "config file (default is cannon.yaml)")
+
+	for _, override := range CannonOverrides {
+		override.FlagHelper(cannonCmd)
+	}
 }
 
 func loadcannonConfigFromFile(file string) (*cannon.Config, error) {

diff --git a/cmd/cl-mimicry.go b/cmd/cl-mimicry.go
@@ -6,7 +6,6 @@ import (
 
 	"github.com/creasty/defaults"
 	"github.com/ethpandaops/xatu/pkg/clmimicry"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -24,21 +23,14 @@ var clMimicryCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		initCommon()
 
-		log.WithField("location", clMimicryCfgFile).Info("Loading config")
-
 		config, err := loadCLMimicryConfigFromFile(clMimicryCfgFile)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		log.Info("Config loaded")
-
-		logLevel, err := logrus.ParseLevel(config.LoggingLevel)
-		if err != nil {
-			log.WithField("logLevel", config.LoggingLevel).Fatal("invalid logging level")
-		}
+		log = getLogger(config.LoggingLevel, "")
 
-		log.SetLevel(logLevel)
+		log.WithField("location", clMimicryCfgFile).Info("Loaded config")
 
 		mimicry, err := clmimicry.New(cmd.Context(), log, config)
 		if err != nil {

diff --git a/cmd/discovery.go b/cmd/discovery.go
@@ -6,7 +6,6 @@ import (
 
 	"github.com/creasty/defaults"
 	"github.com/ethpandaops/xatu/pkg/discovery"
-	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -24,22 +23,14 @@ var discoveryCmd = &cobra.Command{
 	Run: func(cmd *cobra.Command, args []string) {
 		initCommon()
 
-		log.WithField("location", discoveryCfgFile).Info("Loading config")
-
 		config, err := loadDiscoveryConfigFromFile(discoveryCfgFile)
 		if err != nil {
 			log.Fatal(err)
 		}
 
-		log.Info("Config loaded")
-
-		logLevel, err := logrus.ParseLevel(config.LoggingLevel)
-		if err != nil {
-			log.WithField("logLevel", config.LoggingLevel).Fatal("invalid logging level")
-		}
-
-		log.SetLevel(logLevel)
+		log = getLogger(config.LoggingLevel, "")
 
+		log.WithField("location", discoveryCfgFile).Info("Loaded config")
 		discovery, err := discovery.New(cmd.Context(), log, config)
 		if err != nil {
 			log.Fatal(err)