Merge pull request #1645 from gobitfly/staging

merge staging to master
gobitfly · Nov 4, 2022 · 24425a3 · 24425a3
2 parents de0a0d8 + d710b96
commit 24425a3
Show file tree

Hide file tree

Showing 10 changed files with 69 additions and 27 deletions.
diff --git a/cmd/explorer/main.go b/cmd/explorer/main.go
@@ -104,7 +104,7 @@ func main() {
 			Name:     cfg.Frontend.ReaderDatabase.Name,
 			Host:     cfg.Frontend.ReaderDatabase.Host,
 			Port:     cfg.Frontend.ReaderDatabase.Port,
-		}, cfg.Frontend.SessionSecret)
+		})
 	}()
 
 	wg.Add(1)

diff --git a/cmd/statistics/main.go b/cmd/statistics/main.go
@@ -48,6 +48,22 @@ func main() {
 	defer db.ReaderDb.Close()
 	defer db.WriterDb.Close()
 
+	db.MustInitFrontendDB(&types.DatabaseConfig{
+		Username: cfg.Frontend.WriterDatabase.Username,
+		Password: cfg.Frontend.WriterDatabase.Password,
+		Name:     cfg.Frontend.WriterDatabase.Name,
+		Host:     cfg.Frontend.WriterDatabase.Host,
+		Port:     cfg.Frontend.WriterDatabase.Port,
+	}, &types.DatabaseConfig{
+		Username: cfg.Frontend.ReaderDatabase.Username,
+		Password: cfg.Frontend.ReaderDatabase.Password,
+		Name:     cfg.Frontend.ReaderDatabase.Name,
+		Host:     cfg.Frontend.ReaderDatabase.Host,
+		Port:     cfg.Frontend.ReaderDatabase.Port,
+	})
+	defer db.FrontendReaderDB.Close()
+	defer db.FrontendWriterDB.Close()
+
 	db.InitBigtable(cfg.Bigtable.Project, cfg.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID))
 
 	if *statisticsDaysToExport != "" {
@@ -101,6 +117,15 @@ func main() {
 
 func statisticsLoop() {
 	for {
+
+		// create stats parition on users table
+		now := time.Now()
+		nowTs := now.Unix()
+		var day int = int(nowTs / 86400)
+
+		db.CreateNewStatsMetaPartition(day)
+		db.CreateNewStatsMetaPartition(day + 1)
+
 		latestEpoch, err := db.GetLatestEpoch()
 		if err != nil {
 			logrus.Errorf("error retreiving latest epoch from the db: %v", err)

diff --git a/db/frontend.go b/db/frontend.go
@@ -20,7 +20,7 @@ import (
 var FrontendReaderDB *sqlx.DB
 var FrontendWriterDB *sqlx.DB
 
-func MustInitFrontendDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, sessionSecret string) {
+func MustInitFrontendDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig) {
 	FrontendWriterDB, FrontendReaderDB = mustInitDB(writer, reader)
 }
 
@@ -618,37 +618,41 @@ func InsertStatsMeta(tx *sql.Tx, userID uint64, data *types.StatsMeta) (uint64,
 	return id, err
 }
 
-func CreateNewStatsMetaPartition() error {
-
-	now := time.Now()
-	nowTs := now.Unix()
-	var day int = int(nowTs / 86400)
-
+func CreateNewStatsMetaPartition(day int) error {
 	partitionName := "stats_meta_" + strconv.Itoa(day)
-	logger.Info("creating new partition table " + partitionName)
 
-	tx, err := FrontendWriterDB.Begin()
+	tx, err := FrontendWriterDB.Beginx()
 	if err != nil {
 		logger.Errorf("error starting tx for creating partition %v", err)
 		return err
 	}
 	defer tx.Rollback()
 
-	_, err = tx.Exec("CREATE TABLE " + partitionName + " PARTITION OF stats_meta_p FOR VALUES IN (" + strconv.Itoa(day) + ")")
+	var exists bool
+	err = tx.Get(&exists, `SELECT EXISTS (SELECT FROM pg_tables WHERE  schemaname = 'public' AND tablename  = $1);`, partitionName)
 	if err != nil {
-		logger.Errorf("error creating partition %v", err)
-		return err
-	}
-	_, err = tx.Exec("CREATE UNIQUE INDEX " + partitionName + "_user_id_created_trunc_process_machine_key ON public." + partitionName + " USING btree (user_id, created_trunc, process, machine)")
-	if err != nil {
-		logger.Errorf("error creating index %v", err)
+		logger.Errorf("error checking if partition %s already exists: %v", partitionName, err)
 		return err
 	}
 
-	_, err = tx.Exec("CREATE INDEX CONCURRENTLY idx_" + partitionName + "_user_machine ON " + partitionName + " (user_id, machine)")
-	if err != nil {
-		logger.Errorf("error creating index %v", err)
-		return err
+	if !exists {
+		logger.Infof("creating partition %s", partitionName)
+		_, err = tx.Exec("CREATE TABLE " + partitionName + " PARTITION OF stats_meta_p FOR VALUES IN (" + strconv.Itoa(day) + ")")
+		if err != nil {
+			logger.Errorf("error creating partition %v", err)
+			return err
+		}
+		_, err = tx.Exec("CREATE UNIQUE INDEX " + partitionName + "_user_id_created_trunc_process_machine_key ON public." + partitionName + " USING btree (user_id, created_trunc, process, machine)")
+		if err != nil {
+			logger.Errorf("error creating index %v", err)
+			return err
+		}
+
+		_, err = tx.Exec("CREATE INDEX idx_" + partitionName + "_user_machine ON " + partitionName + " (user_id, machine)")
+		if err != nil {
+			logger.Errorf("error creating index %v", err)
+			return err
+		}
 	}
 
 	err = tx.Commit()

diff --git a/go.mod b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/ethereum/go-ethereum v1.10.23
 	github.com/evanw/esbuild v0.8.23
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/gobitfly/eth-rewards v0.1.2-0.20221102075232-f1fad9916be8
+	github.com/gobitfly/eth-rewards v0.1.2-0.20221103132515-d5e7fc3a781c
 	github.com/gobitfly/eth.store v0.0.0-20221012115129-3b4606992669
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/golang/protobuf v1.5.2

diff --git a/go.sum b/go.sum
@@ -476,6 +476,10 @@ github.com/gobitfly/eth-rewards v0.1.2-0.20221102074626-2f0a040aade5 h1:QA+RnPmj
 github.com/gobitfly/eth-rewards v0.1.2-0.20221102074626-2f0a040aade5/go.mod h1:GEerfGH16ByR90QoS/9QyJ0UyCHQRLQ9QH6ZbfneSdU=
 github.com/gobitfly/eth-rewards v0.1.2-0.20221102075232-f1fad9916be8 h1:WKS/xtcRroGdnennVoW97un2O9kOHxetVwQ0w633R50=
 github.com/gobitfly/eth-rewards v0.1.2-0.20221102075232-f1fad9916be8/go.mod h1:GEerfGH16ByR90QoS/9QyJ0UyCHQRLQ9QH6ZbfneSdU=
+github.com/gobitfly/eth-rewards v0.1.2-0.20221103131949-87d1f7df9638 h1:rq/PeVbmxrhwc0WXUI6K2AyreH55Ldm6+ErqSqFmAZM=
+github.com/gobitfly/eth-rewards v0.1.2-0.20221103131949-87d1f7df9638/go.mod h1:GEerfGH16ByR90QoS/9QyJ0UyCHQRLQ9QH6ZbfneSdU=
+github.com/gobitfly/eth-rewards v0.1.2-0.20221103132515-d5e7fc3a781c h1:43NH7kVRmeKaUfVAHevD00hnDEzOSctuEnM4M21RagU=
+github.com/gobitfly/eth-rewards v0.1.2-0.20221103132515-d5e7fc3a781c/go.mod h1:GEerfGH16ByR90QoS/9QyJ0UyCHQRLQ9QH6ZbfneSdU=
 github.com/gobitfly/eth.store v0.0.0-20221012115129-3b4606992669 h1:nGqtKWpeItBrYO0TkdSSpeVRpjcWW6il28EasO9WZSo=
 github.com/gobitfly/eth.store v0.0.0-20221012115129-3b4606992669/go.mod h1:NlB9l6tiM42FM5qQHIsG/Iey6AJ0XQvRcKkmt63/ve0=
 github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=

diff --git a/handlers/api.go b/handlers/api.go
@@ -2336,7 +2336,11 @@ func insertStats(userData *types.UserWithPremium, machine string, body *map[stri
 		if strings.Contains(err.Error(), "no partition of relation") {
 			tx.Rollback()
 
-			db.CreateNewStatsMetaPartition()
+			now := time.Now()
+			nowTs := now.Unix()
+			var day int = int(nowTs / 86400)
+			db.CreateNewStatsMetaPartition(day)
+
 			tx, err = db.NewTransaction()
 			if err != nil {
 				logger.Errorf("could not transact | %v", err)

diff --git a/handlers/eth1Block.go b/handlers/eth1Block.go
@@ -91,6 +91,7 @@ func Eth1Block(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		blockPageData.ExecutionData = eth1BlockPageData
+		blockPageData.ExecutionData.IsValidMev = blockPageData.IsValidMev
 
 		data.HeaderAd = true
 		data.Data = blockPageData

diff --git a/handlers/eth1Deposits.go b/handlers/eth1Deposits.go
@@ -34,7 +34,7 @@ func Eth1Deposits(w http.ResponseWriter, r *http.Request) {
 	pageData.Stats = services.GetLatestStats()
 	pageData.DepositContract = utils.Config.Indexer.Eth1DepositContractAddress
 
-	data := InitPageData(w, r, "blockchain", "/deposits/eth1", "Eth1 Deposits")
+	data := InitPageData(w, r, "blockchain", "/deposits/eth1", "Initiated Deposits")
 	data.HeaderAd = true
 	data.Data = pageData
 
@@ -148,7 +148,7 @@ func Eth1DepositsLeaderboard(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "text/html")
 
-	data := InitPageData(w, r, "eth1Deposits", "/deposits/eth1", "Eth1 Deposits")
+	data := InitPageData(w, r, "eth1Deposits", "/deposits/eth1", "Initiated Deposits")
 	data.HeaderAd = true
 
 	data.Data = types.EthOneDepositLeaderBoardPageData{

diff --git a/handlers/eth2Deposits.go b/handlers/eth2Deposits.go
@@ -18,7 +18,7 @@ func Eth2Deposits(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "text/html")
 
-	data := InitPageData(w, r, "eth2Deposits", "/deposits/eth2", "Eth2 Deposits")
+	data := InitPageData(w, r, "eth2Deposits", "/deposits/eth2", "Included Deposits")
 	data.HeaderAd = true
 
 	err := eth2DepositsTemplate.ExecuteTemplate(w, "layout", data)

diff --git a/services/notifications.go b/services/notifications.go
@@ -160,11 +160,12 @@ func notificationsSender() {
 func notificationSender() {
 	for {
 		start := time.Now()
-		ctx, _ := context.WithTimeout(context.Background(), time.Second*30)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
 
 		conn, err := db.FrontendReaderDB.Conn(ctx)
 		if err != nil {
 			logger.WithError(err).Error("error creating connection")
+			cancel()
 			continue
 		}
 
@@ -176,6 +177,7 @@ func notificationSender() {
 			if err != nil {
 				logger.WithError(err).Error("error returning connection to connection pool")
 			}
+			cancel()
 			continue
 		}
 
@@ -200,6 +202,7 @@ func notificationSender() {
 			if err != nil {
 				logger.WithError(err).Error("error returning connection to connection pool")
 			}
+			cancel()
 			continue
 		}
 
@@ -215,6 +218,7 @@ func notificationSender() {
 		if err != nil {
 			logger.WithError(err).Error("error returning connection to connection pool")
 		}
+		cancel()
 
 		time.Sleep(time.Second * 30)
 	}