Skip to content

Commit

Permalink
Get latest updates from upstream and solve conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
Giacomo Licari committed Nov 4, 2022
2 parents cf5e874 + 24425a3 commit 480f7ba
Show file tree
Hide file tree
Showing 70 changed files with 1,761 additions and 1,025 deletions.
36 changes: 36 additions & 0 deletions .github/ISSUE_TEMPLATE/bug_report.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''

---

**Describe the bug**
A clear and concise description of what the bug is.

**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error

**Expected behavior**
A clear and concise description of what you expected to happen.

**Screenshots**
If applicable, add screenshots to help explain your problem.

**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]

**Phone (please complete the following information):**
- OS: [e.g. iOS 14.5]
- App Version [e.g. 22]

**Additional context**
Add any other context about the problem here.
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ BUILDDATE=`date -u +"%Y-%m-%dT%H:%M:%S%:z"`
PACKAGE=eth2-exporter
LDFLAGS="-X ${PACKAGE}/version.Version=${VERSION} -X ${PACKAGE}/version.BuildDate=${BUILDDATE} -X ${PACKAGE}/version.GitCommit=${GITCOMMIT} -X ${PACKAGE}/version.GitDate=${GITDATE} -s -w"
all: explorer stats frontend-data-updater eth1indexer ethstore-exporter
all: explorer stats frontend-data-updater eth1indexer ethstore-exporter rewards-exporter
lint:
golint ./...
Expand All @@ -29,5 +29,8 @@ frontend-data-updater:
ethstore-exporter:
go build --ldflags=${LDFLAGS} -o bin/ethstore-exporter cmd/ethstore-exporter/main.go
rewards-exporter:
go build --ldflags=${LDFLAGS} -o bin/rewards-exporter cmd/rewards-exporter/main.go
eth1indexer:
go build --ldflags=${LDFLAGS} -o bin/eth1indexer cmd/eth1indexer/main.go
20 changes: 14 additions & 6 deletions cmd/eth1indexer/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"eth2-exporter/erc20"
"eth2-exporter/rpc"
"eth2-exporter/types"
"eth2-exporter/version"
"flag"
"fmt"
"io/ioutil"
Expand Down Expand Up @@ -51,17 +52,23 @@ func main() {

enableBalanceUpdater := flag.Bool("balances.enabled", false, "Enable balance update process")
enableFullBalanceUpdater := flag.Bool("balances.full.enabled", false, "Enable full balance update process")
balanceUpdaterPrefix := flag.String("balances.prefix", "", "Prefix to use for fetching balance updates")
balanceUpdaterBatchSize := flag.Int("balances.batch", 1000, "Batch size for balance updates")

tokenPriceExport := flag.Bool("token.price.enabled", false, "Enable token export process")
tokenPriceExportList := flag.String("token.price.list", "", "Tokenlist path to use for the token price export")
tokenPriceExportFrequency := flag.Duration("token.price.frequency", time.Hour, "Token price export interval")

bigtableProject := flag.String("bigtable.project", "", "ID of the BigTable project")
bigtableInstance := flag.String("bigtable.instance", "", "ID of the BigTable instance")
bigtableProject := flag.String("bigtable.project", "", "Bigtable project")
bigtableInstance := flag.String("bigtable.instance", "", "Bigtable instance")

versionFlag := flag.Bool("version", false, "Print version and exit")

flag.Parse()
if *versionFlag {
fmt.Println(version.Version)
return
}

if erigonEndpoint == nil || *erigonEndpoint == "" {
logrus.Fatal("no erigon node url provided")
}
Expand Down Expand Up @@ -90,6 +97,7 @@ func main() {
} else {
logrus.Fatalf("unsupported network name %v provided", *network)
}
balanceUpdaterPrefix := chainId + ":B:"

nodeChainId, err := client.GetNativeClient().ChainID(context.Background())
if err != nil {
Expand Down Expand Up @@ -124,9 +132,9 @@ func main() {
// }
// return
if *enableFullBalanceUpdater {
ProcessMetadataUpdates(bt, client, *balanceUpdaterPrefix, *balanceUpdaterBatchSize, -1)
ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, -1)
return
// currentKey := *balanceUpdaterPrefix // "1:00028ebf7d36c5779c1deddf3ba72761fd46c8aa"
// currentKey := balanceUpdaterPrefix // "1:00028ebf7d36c5779c1deddf3ba72761fd46c8aa"
// for {
// keys, pairs, err := bt.GetMetadata(currentKey, *balanceUpdaterBatchSize)
// if err != nil {
Expand Down Expand Up @@ -250,7 +258,7 @@ func main() {
}

if *enableBalanceUpdater {
ProcessMetadataUpdates(bt, client, *balanceUpdaterPrefix, *balanceUpdaterBatchSize, 10)
ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, 10)
}

logrus.Infof("index run completed")
Expand Down
18 changes: 9 additions & 9 deletions cmd/explorer/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func main() {
Name: cfg.Frontend.ReaderDatabase.Name,
Host: cfg.Frontend.ReaderDatabase.Host,
Port: cfg.Frontend.ReaderDatabase.Port,
}, cfg.Frontend.SessionSecret)
})
}()

wg.Add(1)
Expand Down Expand Up @@ -249,6 +249,7 @@ func main() {
apiV1Router.HandleFunc("/validator/leaderboard", handlers.ApiValidatorLeaderboard).Methods("GET", "OPTIONS")
apiV1Router.HandleFunc("/validator/{indexOrPubkey}", handlers.ApiValidator).Methods("GET", "OPTIONS")
apiV1Router.HandleFunc("/validator/{indexOrPubkey}/balancehistory", handlers.ApiValidatorBalanceHistory).Methods("GET", "OPTIONS")
apiV1Router.HandleFunc("/validator/{indexOrPubkey}/incomedetailhistory", handlers.ApiValidatorIncomeDetailsHistory).Methods("GET", "OPTIONS")
apiV1Router.HandleFunc("/validator/{indexOrPubkey}/performance", handlers.ApiValidatorPerformance).Methods("GET", "OPTIONS")
apiV1Router.HandleFunc("/validator/{indexOrPubkey}/execution/performance", handlers.ApiValidatorExecutionPerformance).Methods("GET", "OPTIONS")
apiV1Router.HandleFunc("/validator/{indexOrPubkey}/attestations", handlers.ApiValidatorAttestations).Methods("GET", "OPTIONS")
Expand Down Expand Up @@ -395,12 +396,12 @@ func main() {
router.HandleFunc("/validators/leaderboard/data", handlers.ValidatorsLeaderboardData).Methods("GET")
router.HandleFunc("/validators/streakleaderboard", handlers.ValidatorsStreakLeaderboard).Methods("GET")
router.HandleFunc("/validators/streakleaderboard/data", handlers.ValidatorsStreakLeaderboardData).Methods("GET")
router.HandleFunc("/validators/eth1deposits", handlers.Eth1Deposits).Methods("GET")
router.HandleFunc("/validators/eth1deposits/data", handlers.Eth1DepositsData).Methods("GET")
router.HandleFunc("/validators/eth1leaderboard", handlers.Eth1DepositsLeaderboard).Methods("GET")
router.HandleFunc("/validators/eth1leaderboard/data", handlers.Eth1DepositsLeaderboardData).Methods("GET")
router.HandleFunc("/validators/eth2deposits", handlers.Eth2Deposits).Methods("GET")
router.HandleFunc("/validators/eth2deposits/data", handlers.Eth2DepositsData).Methods("GET")
router.HandleFunc("/validators/initiated-deposits", handlers.Eth1Deposits).Methods("GET")
router.HandleFunc("/validators/initiated-deposits/data", handlers.Eth1DepositsData).Methods("GET")
router.HandleFunc("/validators/deposit-leaderboard", handlers.Eth1DepositsLeaderboard).Methods("GET")
router.HandleFunc("/validators/deposit-leaderboard/data", handlers.Eth1DepositsLeaderboardData).Methods("GET")
router.HandleFunc("/validators/included-deposits", handlers.Eth2Deposits).Methods("GET")
router.HandleFunc("/validators/included-deposits/data", handlers.Eth2DepositsData).Methods("GET")

router.HandleFunc("/dashboard", handlers.Dashboard).Methods("GET")
router.HandleFunc("/dashboard/save", handlers.UserDashboardWatchlistAdd).Methods("POST")
Expand All @@ -417,12 +418,11 @@ func main() {
router.HandleFunc("/search", handlers.Search).Methods("POST")
router.HandleFunc("/search/{type}/{search}", handlers.SearchAhead).Methods("GET")
router.HandleFunc("/faq", handlers.Faq).Methods("GET")
router.HandleFunc("/imprint", handlers.Imprint).Methods("GET")
// router.HandleFunc("/imprint", handlers.Imprint).Methods("GET")
router.HandleFunc("/poap", handlers.Poap).Methods("GET")
router.HandleFunc("/poap/data", handlers.PoapData).Methods("GET")
router.HandleFunc("/mobile", handlers.MobilePage).Methods("GET")
router.HandleFunc("/mobile", handlers.MobilePagePost).Methods("POST")

router.HandleFunc("/tools/unitConverter", handlers.UnitConverter).Methods("GET")

router.HandleFunc("/tables/state", handlers.DataTableStateChanges).Methods("POST")
Expand Down
107 changes: 107 additions & 0 deletions cmd/rewards-exporter/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package main

import (
"eth2-exporter/db"
"eth2-exporter/rpc"
"eth2-exporter/types"
"eth2-exporter/utils"
"eth2-exporter/version"
"flag"
"fmt"
"math/big"
"time"

geth_rpc "github.com/ethereum/go-ethereum/rpc"

eth_rewards "github.com/gobitfly/eth-rewards"
"github.com/prysmaticlabs/prysm/v3/api/client/beacon"
"github.com/sirupsen/logrus"
)

func main() {

configPath := flag.String("config", "", "Path to the config file, if empty string defaults will be used")
bnAddress := flag.String("beacon-node-address", "", "Url of the beacon node api")
enAddress := flag.String("execution-node-address", "", "Url of the execution node api")
epoch := flag.Int64("epoch", -1, "epoch to export (use -1 to export latest finalized epoch)")
network := flag.String("network", "", "Config to use (can be mainnet, prater or sepolia")

flag.Parse()

cfg := &types.Config{}
err := utils.ReadConfig(cfg, *configPath)
if err != nil {
logrus.Fatalf("error reading config file: %v", err)
}
utils.Config = cfg
logrus.WithField("config", *configPath).WithField("version", version.Version).WithField("chainName", utils.Config.Chain.Config.ConfigName).Printf("starting")

client, err := beacon.NewClient(*bnAddress)
if err != nil {
logrus.Fatal(err)
}

lc, err := rpc.NewLighthouseClient(*bnAddress, big.NewInt(5))
if err != nil {
logrus.Fatal(err)
}

elClient, err := geth_rpc.Dial(*enAddress)
if err != nil {
logrus.Fatal(err)
}

bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID))
if err != nil {
logrus.Fatalf("error connecting to bigtable: %v", err)
}
defer bt.Close()

if *epoch == -1 {
for {
head, err := lc.GetChainHead()
if err != nil {
logrus.Fatal(err)
}
if int64(head.FinalizedEpoch) <= *epoch {
time.Sleep(time.Second * 12)
continue
}

*epoch = int64(head.FinalizedEpoch)

start := time.Now()
logrus.Infof("retrieving rewards details for epoch %v", *epoch)

rewards, err := eth_rewards.GetRewardsForEpoch(int(*epoch), client, elClient, *network)

if err != nil {
logrus.Fatalf("error retrieving reward details for epoch %v: %v", *epoch, err)
} else {
logrus.Infof("retrieved %v reward details for epoch %v in %v", len(rewards), *epoch, time.Since(start))
}

err = bt.SaveValidatorIncomeDetails(uint64(*epoch), rewards)
if err != nil {
logrus.Fatalf("error saving reward details to bigtable: %v", err)
}
}
}

start := time.Now()
logrus.Infof("retrieving rewards details for epoch %v", *epoch)

rewards, err := eth_rewards.GetRewardsForEpoch(int(*epoch), client, elClient, *network)

if err != nil {
logrus.Fatalf("error retrieving reward details for epoch %v: %v", *epoch, err)
} else {
logrus.Infof("retrieved %v reward details for epoch %v in %v", len(rewards), *epoch, time.Since(start))
}

err = bt.SaveValidatorIncomeDetails(uint64(*epoch), rewards)
if err != nil {
logrus.Fatalf("error saving reward details to bigtable: %v", err)
}

}
25 changes: 25 additions & 0 deletions cmd/statistics/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,22 @@ func main() {
defer db.ReaderDb.Close()
defer db.WriterDb.Close()

db.MustInitFrontendDB(&types.DatabaseConfig{
Username: cfg.Frontend.WriterDatabase.Username,
Password: cfg.Frontend.WriterDatabase.Password,
Name: cfg.Frontend.WriterDatabase.Name,
Host: cfg.Frontend.WriterDatabase.Host,
Port: cfg.Frontend.WriterDatabase.Port,
}, &types.DatabaseConfig{
Username: cfg.Frontend.ReaderDatabase.Username,
Password: cfg.Frontend.ReaderDatabase.Password,
Name: cfg.Frontend.ReaderDatabase.Name,
Host: cfg.Frontend.ReaderDatabase.Host,
Port: cfg.Frontend.ReaderDatabase.Port,
})
defer db.FrontendReaderDB.Close()
defer db.FrontendWriterDB.Close()

db.InitBigtable(cfg.Bigtable.Project, cfg.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID))

if *statisticsDaysToExport != "" {
Expand Down Expand Up @@ -101,6 +117,15 @@ func main() {

func statisticsLoop() {
for {

// create stats parition on users table
now := time.Now()
nowTs := now.Unix()
var day int = int(nowTs / 86400)

db.CreateNewStatsMetaPartition(day)
db.CreateNewStatsMetaPartition(day + 1)

latestEpoch, err := db.GetLatestEpoch()
if err != nil {
logrus.Errorf("error retreiving latest epoch from the db: %v", err)
Expand Down
Loading

0 comments on commit 480f7ba

Please sign in to comment.