Add address checksum validation for tokenlist files (trustwallet#17416)

* Add address checksum validation for tokenlist files

* Remove updaters_manual.go, move tokenlist validation to assets-go-libs

README.md

@@ -57,7 +57,6 @@ There are several scripts available for maintainers:
 - `make check` -- Execute validation checks; also used in continuous integration.
 - `make fix` -- Perform automatic fixes where possible
 - `make update-auto` -- Run automatic updates from external sources, executed regularly (GitHub action)
-- `make update-manual` -- Run manual updates from external sources, for manual use.
 - `make add-token token=c60_t0x4Fabb145d64652a948d72533023f6E7A623C7C53` -- Create `info.json` file as asset template.
 
 ## On Checks

go.mod

@@ -5,7 +5,7 @@ go 1.17
 require (
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.3.0
-	github.com/trustwallet/assets-go-libs v0.0.25
+	github.com/trustwallet/assets-go-libs v0.0.26
 	github.com/trustwallet/go-libs v0.2.23
 	github.com/trustwallet/go-primitives v0.0.20
 )

go.sum

@@ -340,8 +340,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/trustwallet/assets-go-libs v0.0.25 h1:JoBZnKOHpHp1pmFjLqWcdlGE1xgGDQUYgHmYIOCZGnY=
-github.com/trustwallet/assets-go-libs v0.0.25/go.mod h1:UKacopV6UfT2JKRO4mkNSOVij9ChOO/NOXLscd9VhLk=
+github.com/trustwallet/assets-go-libs v0.0.26 h1:CtqSfhu/sNV/q256977UF6niIabmEvzrJVUwD4GJlDg=
+github.com/trustwallet/assets-go-libs v0.0.26/go.mod h1:UKacopV6UfT2JKRO4mkNSOVij9ChOO/NOXLscd9VhLk=
 github.com/trustwallet/go-libs v0.2.23 h1:CUB2OubedAoUbgPcraIC7wqpJHKQ5o3NJdINzH418WQ=
 github.com/trustwallet/go-libs v0.2.23/go.mod h1:7QdAp1lcteKKI0DYqGoaO8KO4eTNYjGmg8vHy0YXkKc=
 github.com/trustwallet/go-primitives v0.0.20 h1:srXOScQzub2Usrj5mRq5d2/3+0Y3TBXh2wD6JXYH1No=

internal/manager/commands.go

@@ -23,7 +23,6 @@ func InitCommands() {
 	rootCmd.AddCommand(checkCmd)
 	rootCmd.AddCommand(fixCmd)
 	rootCmd.AddCommand(updateAutoCmd)
-	rootCmd.AddCommand(updateManualCmd)
 	rootCmd.AddCommand(addTokenCmd)
 }
 
@@ -58,14 +57,6 @@ var (
 			assetsService.RunUpdateAuto()
 		},
 	}
-	updateManualCmd = &cobra.Command{
-		Use:   "update-manual",
-		Short: "Run manual updates from external sources",
-		Run: func(cmd *cobra.Command, args []string) {
-			assetsService := InitAssetsService()
-			assetsService.RunUpdateManual()
-		},
-	}
 
 	addTokenCmd = &cobra.Command{
 		Use:   "add-token",

internal/processor/model.go

@@ -2,7 +2,6 @@ package processor
 
 import (
 	"github.com/trustwallet/assets/internal/file"
-	"github.com/trustwallet/go-primitives/types"
 )
 
 type (
@@ -22,39 +21,6 @@ type (
 	}
 )
 
-type (
-	TokenList struct {
-		Name      string      `json:"name"`
-		LogoURI   string      `json:"logoURI"`
-		Timestamp string      `json:"timestamp"`
-		Tokens    []TokenItem `json:"tokens"`
-		Version   Version     `json:"version"`
-	}
-
-	TokenItem struct {
-		Asset    string          `json:"asset"`
-		Type     types.TokenType `json:"type"`
-		Address  string          `json:"address"`
-		Name     string          `json:"name"`
-		Symbol   string          `json:"symbol"`
-		Decimals uint            `json:"decimals"`
-		LogoURI  string          `json:"logoURI"`
-		Pairs    []Pair          `json:"pairs"`
-	}
-
-	Pair struct {
-		Base     string `json:"base"`
-		LotSize  string `json:"lotSize,omitempty"`
-		TickSize string `json:"tickSize,omitempty"`
-	}
-
-	Version struct {
-		Major int `json:"major"`
-		Minor int `json:"minor"`
-		Patch int `json:"patch"`
-	}
-)
-
 type (
 	ForceListPair struct {
 		Token0 string

internal/processor/service.go

@@ -108,10 +108,3 @@ func (s *Service) GetUpdatersAuto() []Updater {
 		{Name: "Retrieving missing token images, creating binance token list.", Run: s.UpdateBinanceTokens},
 	}
 }
-
-func (s *Service) GetUpdatersManual() []Updater {
-	return []Updater{
-		{Name: "Update tokenlist.json for Ethereum", Run: s.UpdateEthereumTokenlist},
-		{Name: "Update tokenlist.json for Smartchain", Run: s.UpdateSmartchainTokenlist},
-	}
-}

internal/processor/updaters_auto.go

@@ -16,6 +16,7 @@ import (
 	"github.com/trustwallet/assets-go-libs/image"
 	"github.com/trustwallet/assets-go-libs/path"
 	"github.com/trustwallet/assets-go-libs/validation/info"
+	"github.com/trustwallet/assets-go-libs/validation/tokenlist"
 	"github.com/trustwallet/assets/internal/config"
 	"github.com/trustwallet/go-libs/blockchain/binance"
 	"github.com/trustwallet/go-libs/blockchain/binance/explorer"
@@ -137,10 +138,10 @@ func createInfoJSON(chain coin.Coin, a explorer.Bep2Asset) error {
 	return fileLib.CreateJSONFile(assetInfoPath, &assetInfo)
 }
 
-func createTokenListJSON(chain coin.Coin, tokens []TokenItem) error {
+func createTokenListJSON(chain coin.Coin, tokens []tokenlist.Token) error {
 	tokenListPath := path.GetTokenListPath(chain.Handle)
 
-	var oldTokenList TokenList
+	var oldTokenList tokenlist.Model
 	err := fileLib.ReadJSONFile(tokenListPath, &oldTokenList)
 	if err != nil {
 		return nil
@@ -157,16 +158,16 @@ func createTokenListJSON(chain coin.Coin, tokens []TokenItem) error {
 	log.Debugf("Tokenlist: list with %d tokens and %d pairs written to %s.",
 		len(tokens), countTotalPairs(tokens), tokenListPath)
 
-	return fileLib.CreateJSONFile(tokenListPath, &TokenList{
+	return fileLib.CreateJSONFile(tokenListPath, &tokenlist.Model{
 		Name:      fmt.Sprintf("Trust Wallet: %s", coin.Coins[chain.ID].Name),
 		LogoURI:   twLogoURL,
 		Timestamp: time.Now().Format(timestampFormat),
 		Tokens:    tokens,
-		Version:   Version{Major: oldTokenList.Version.Major + 1},
+		Version:   tokenlist.Version{Major: oldTokenList.Version.Major + 1},
 	})
 }
 
-func countTotalPairs(tokens []TokenItem) int {
+func countTotalPairs(tokens []tokenlist.Token) int {
 	var counter int
 	for _, token := range tokens {
 		counter += len(token.Pairs)
@@ -175,7 +176,7 @@ func countTotalPairs(tokens []TokenItem) int {
 	return counter
 }
 
-func sortTokens(tokens []TokenItem) {
+func sortTokens(tokens []tokenlist.Token) {
 	sort.Slice(tokens, func(i, j int) bool {
 		if len(tokens[i].Pairs) != len(tokens[j].Pairs) {
 			return len(tokens[i].Pairs) > len(tokens[j].Pairs)
@@ -191,7 +192,7 @@ func sortTokens(tokens []TokenItem) {
 	}
 }
 
-func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Tokens) ([]TokenItem, error) {
+func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Tokens) ([]tokenlist.Token, error) {
 	if len(marketPairs) < 5 {
 		return nil, fmt.Errorf("no markets info is returned from Binance DEX: %d", len(marketPairs))
 	}
@@ -200,7 +201,7 @@ func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Token
 		return nil, fmt.Errorf("no tokens info is returned from Binance DEX: %d", len(tokenList))
 	}
 
-	pairsMap := make(map[string][]Pair)
+	pairsMap := make(map[string][]tokenlist.Pair)
 	pairsList := make(map[string]struct{})
 	tokensMap := make(map[string]binance.Token)
 
@@ -219,25 +220,25 @@ func generateTokenList(marketPairs []binance.MarketPair, tokenList binance.Token
 			val = append(val, getPair(marketPair))
 			pairsMap[tokenSymbol] = val
 		} else {
-			pairsMap[tokenSymbol] = []Pair{getPair(marketPair)}
+			pairsMap[tokenSymbol] = []tokenlist.Pair{getPair(marketPair)}
 		}
 
 		pairsList[marketPair.BaseAssetSymbol] = struct{}{}
 		pairsList[marketPair.QuoteAssetSymbol] = struct{}{}
 	}
 
-	tokenItems := make([]TokenItem, 0, len(pairsList))
+	tokenItems := make([]tokenlist.Token, 0, len(pairsList))
 
 	for pair := range pairsList {
 		token := tokensMap[pair]
 
-		var pairs []Pair
+		var pairs []tokenlist.Pair
 		pairs, exists := pairsMap[token.Symbol]
 		if !exists {
-			pairs = make([]Pair, 0)
+			pairs = make([]tokenlist.Pair, 0)
 		}
 
-		tokenItems = append(tokenItems, TokenItem{
+		tokenItems = append(tokenItems, tokenlist.Token{
 			Asset:    getAssetIDSymbol(token.Symbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE),
 			Type:     getTokenType(token.Symbol, coin.Coins[coin.BINANCE].Symbol, types.BEP2),
 			Address:  token.Symbol,
@@ -288,8 +289,8 @@ func isTokenExistOrActive(symbol string) bool {
 	return true
 }
 
-func getPair(marketPair binance.MarketPair) Pair {
-	return Pair{
+func getPair(marketPair binance.MarketPair) tokenlist.Pair {
+	return tokenlist.Pair{
 		Base:     getAssetIDSymbol(marketPair.BaseAssetSymbol, coin.Coins[coin.BINANCE].Symbol, coin.BINANCE),
 		LotSize:  strconv.FormatInt(numbers.ToSatoshi(marketPair.LotSize), 10),
 		TickSize: strconv.FormatInt(numbers.ToSatoshi(marketPair.TickSize), 10),