chore: improvements on tokens list being used

Instead of returning all tokens (atm there are ~15k unique tokens that the app operates with) from all supported
lists, this change forms a token list based on `token_balances` table which contains tokens for all accounts, for those
tokens includes other tokens that share the same cross chain id and includes all mandatory tokens.

Author: saledjenic

services/wallet/api.go

@@ -151,6 +151,11 @@ func (api *API) GetAllTokens(ctx context.Context) ([]*tokentypes.Token, error) {
 	return api.s.tokenManager.GetAllTokens()
 }
 
+// GetTokensOfInterestForActiveNetworksMode returns all unique tokens that are of interest for the current active networks mode (testnet or mainnet).
+func (api *API) GetTokensOfInterestForActiveNetworksMode(ctx context.Context) ([]*tokentypes.Token, error) {
+	return api.s.tokenManager.GetTokensOfInterestForActiveNetworksMode()
+}
+
 // GetTokensForActiveNetworksMode returns all unique tokens for the current active networks mode (testnet or mainnet).
 func (api *API) GetTokensForActiveNetworksMode(ctx context.Context) ([]*tokentypes.Token, error) {
 	return api.s.tokenManager.GetTokensForActiveNetworksMode()
@@ -326,14 +331,14 @@ func (api *API) GetFlatEthereumChains(ctx context.Context) ([]*params.Network, e
 }
 
 // @deprecated
-// FetchPrices fetches prices for a given token keys and currencies. If no tokens are provided, all tokens are fetched.
+// FetchPrices fetches prices for a given token keys and currencies. If no tokens are provided, all tokens of interest are fetched.
 func (api *API) FetchPrices(ctx context.Context, tokensKeys []string, currencies []string) (map[string]map[string]float64, error) {
 	logutils.ZapLogger().Debug("call to FetchPrices")
 	return api.s.marketManager.FetchPrices(tokensKeys, currencies)
 }
 
 // @deprecated
-// FetchTokenMarketValues fetches market values for a given token keys and currency. If no tokens are provided, all tokens are fetched.
+// FetchTokenMarketValues fetches market values for a given token keys and currency. If no tokens are provided, all tokens of interest are fetched.
 func (api *API) FetchMarketValues(ctx context.Context, tokensKeys []string, currency string) (map[string]thirdparty.TokenMarketValues, error) {
 	logutils.ZapLogger().Debug("call to FetchMarketValues")
 	return api.s.marketManager.FetchTokenMarketValues(tokensKeys, currency)
@@ -350,7 +355,7 @@ func (api *API) GetDailyMarketValues(ctx context.Context, tokenKey string, curre
 }
 
 // @deprecated
-// FetchTokenDetails fetches token details for a given tokens. If no tokens are provided, all tokens are fetched.
+// FetchTokenDetails fetches token details for a given tokens. If no tokens are provided, all tokens of interest are fetched.
 func (api *API) FetchTokenDetails(ctx context.Context, tokensKeys []string) (map[string]thirdparty.TokenDetails, error) {
 	logutils.ZapLogger().Debug("call to FetchTokenDetails")
 	return api.s.marketManager.FetchTokenDetails(tokensKeys)

services/wallet/currency/service.go

@@ -100,7 +100,7 @@ func (s *Service) getAllFiatCurrencyFormats() (FormatPerKey, error) {
 }
 
 func (s *Service) getAllFixedTokenCurrencyFormats() (FormatPerKey, error) {
-	tokens, err := s.tokenManager.GetTokensForActiveNetworksMode()
+	tokens, err := s.tokenManager.GetTokensOfInterestForActiveNetworksMode()
 	if err != nil {
 		return nil, err
 	}
@@ -127,7 +127,7 @@ func (s *Service) getAllFixedTokenCurrencyFormats() (FormatPerKey, error) {
 }
 
 func (s *Service) fetchAllTokenCurrencyFormats() (FormatPerKey, error) {
-	tokens, err := s.tokenManager.GetTokensForActiveNetworksMode()
+	tokens, err := s.tokenManager.GetTokensOfInterestForActiveNetworksMode()
 	if err != nil {
 		return nil, err
 	}

services/wallet/market/market_test.go

@@ -23,7 +23,7 @@ import (
 type MockTokenManager struct{}
 
 func (m *MockTokenManager) GetTokensForFetchingMarketData() ([]*tokentypes.Token, error) {
-	return m.GetTokensForActiveNetworksMode()
+	return m.GetTokensOfInterestForActiveNetworksMode()
 }
 
 func (m *MockTokenManager) GetTokensByKeysForFetchingMarketData(tokenKeys []string) ([]*tokentypes.Token, error) {
@@ -53,7 +53,7 @@ func (m *MockTokenManager) GetTokensByKeys(tokenKeys []string) ([]*tokentypes.To
 	return tokens, nil
 }
 
-func (m *MockTokenManager) GetTokensForActiveNetworksMode() ([]*tokentypes.Token, error) {
+func (m *MockTokenManager) GetTokensOfInterestForActiveNetworksMode() ([]*tokentypes.Token, error) {
 	return m.GetTokensByKeys(testTokensKeys)
 }
 

services/wallet/token/balance_persistence.go

@@ -10,6 +10,9 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 
+	"github.com/status-im/go-wallet-sdk/pkg/tokens/types"
+
+	walletcommon "github.com/status-im/status-go/services/wallet/common"
 	tokentypes "github.com/status-im/status-go/services/wallet/token/types"
 )
 
@@ -182,3 +185,37 @@ func (p *balanceStorage) getCachedBalancesByChain(accounts []common.Address, tok
 
 	return ret, nil
 }
+
+func (p *balanceStorage) getUsedTokensKeys(testnetMode bool) (map[string]interface{}, error) {
+	query := `SELECT token_address, chain_id FROM token_balances`
+	rows, err := p.walletDB.Query(query)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	tokenKeys := make(map[string]interface{}, 0)
+
+	for rows.Next() {
+		var tokenAddressStr string
+		var chainID uint64
+
+		err := rows.Scan(&tokenAddressStr, &chainID)
+		if err != nil {
+			return nil, err
+		}
+
+		if testnetMode && walletcommon.ChainID(chainID).IsMainnet() ||
+			!testnetMode && !walletcommon.ChainID(chainID).IsMainnet() {
+			continue
+		}
+
+		tokenKey := types.TokenKey(chainID, common.HexToAddress(tokenAddressStr))
+
+		if _, ok := tokenKeys[tokenKey]; !ok {
+			tokenKeys[tokenKey] = nil
+		}
+	}
+
+	return tokenKeys, nil
+}

services/wallet/token/token.go

@@ -13,6 +13,8 @@ import (
 
 	"go.uber.org/zap"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/status-im/go-wallet-sdk/pkg/tokens/autofetcher"
 	"github.com/status-im/go-wallet-sdk/pkg/tokens/fetcher"
 	"github.com/status-im/go-wallet-sdk/pkg/tokens/manager"
@@ -389,6 +391,58 @@ func (tm *Manager) GetTokensForActiveNetworksMode() ([]*tokentypes.Token, error)
 	return tm.GetTokensByChains(chainIDs)
 }
 
+// getTokenKeysForTokensOfInterestForActiveNetworksMode returns the token keys for the tokens of interest for the active networks mode (testnet or mainnet)
+// On top of the used tokens keys, it adds all tokens that share the same cross chain id (because of grouping) and all mandatory tokens.
+func (tm *Manager) getTokenKeysForTokensOfInterestForActiveNetworksMode() ([]string, error) {
+	testnetMode, err := tm.settings.GetTestNetworksEnabled()
+	if err != nil {
+		return nil, err
+	}
+
+	usedTokensKeys, err := tm.tokenBalancesStorage.getUsedTokensKeys(testnetMode)
+	if err != nil {
+		return nil, err
+	}
+
+	tokens, err := tm.GetTokensByKeys(maps.Keys(usedTokensKeys))
+	if err != nil {
+		return nil, err
+	}
+
+	// Because of grouping it's important to add all tokens that share the same cross chain id to the used tokens keys
+	for _, token := range tokens {
+		if token.CrossChainID == "" {
+			continue
+		}
+		usedTokensKeys[token.Key()] = nil
+	}
+
+	// It's also important to add all mandatory tokens to the used tokens keys
+	for _, tokenKey := range walletcommon.MandatoryTokens() {
+		if _, ok := usedTokensKeys[tokenKey]; ok {
+			continue
+		}
+		chainID, _, ok := types.ChainAndAddressFromTokenKey(tokenKey)
+		if !ok {
+			continue
+		}
+		if walletcommon.ChainID(chainID).IsMainnet() == testnetMode {
+			continue
+		}
+		usedTokensKeys[tokenKey] = nil
+	}
+
+	return maps.Keys(usedTokensKeys), nil
+}
+
+func (tm *Manager) GetTokensOfInterestForActiveNetworksMode() ([]*tokentypes.Token, error) {
+	tokensKeys, err := tm.getTokenKeysForTokensOfInterestForActiveNetworksMode()
+	if err != nil {
+		return nil, err
+	}
+	return tm.GetTokensByKeys(tokensKeys)
+}
+
 // GetTokensForFetchingMarketData returns all unique tokens for fetching market data from Coingecko (doesn't affect CryptoCompare cause it maps tokens differently, by symbol)
 // Special handling for test tokens, for fetching market data from Coingecko, cause their API doesn't support test tokens
 // Corresponding mainnet tokens are needed to fetch market data for test tokens.
@@ -401,22 +455,16 @@ func (tm *Manager) GetTokensForFetchingMarketData() ([]*tokentypes.Token, error)
 
 	// If not in testnet mode, use the regular tokens
 	if !testnetMode {
-		return tm.GetTokensForActiveNetworksMode()
+		return tm.GetTokensOfInterestForActiveNetworksMode()
 	}
 
 	// Test tokens handling...
 	// Use all test tokens and add to the list only mainnet tokens that have a cross chain id set
-	allWsdkTokens := tm.tokensManager.UniqueTokens()
-	tokens := make([]*tokentypes.Token, 0)
-	for _, token := range allWsdkTokens {
-		if !walletcommon.ChainID(token.ChainID).IsMainnet() {
-			tokens = append(tokens, &tokentypes.Token{Token: token})
-		} else if token.CrossChainID != "" {
-			tokens = append(tokens, &tokentypes.Token{Token: token})
-		}
+	tokensKeys, err := tm.getTokenKeysForTokensOfInterestForActiveNetworksMode()
+	if err != nil {
+		return nil, err
 	}
-
-	return tokens, nil
+	return tm.GetTokensByKeysForFetchingMarketData(tokensKeys)
 }
 
 // GetTokensByKeysForFetchingMarketData returns all unique tokens for fetching market data from Coingecko (doesn't affect CryptoCompare cause it maps tokens differently, by symbol)

tests-unit-network/cryptocompare/cryptocompare_test.go

@@ -64,7 +64,7 @@ func getTokenSymbols(t *testing.T) []*tokentypes.Token {
 	err = tm.Start(context.Background())
 	require.NoError(t, err)
 
-	tokens, err := tm.GetTokensForActiveNetworksMode()
+	tokens, err := tm.GetTokensOfInterestForActiveNetworksMode()
 	require.NoError(t, err)
 	require.Greater(t, len(tokens), 0)