electrum: Satisfy WalletHistorian. (#2843)

* electrum: Satisfy WalletHistorian.

* zec: Ensure balance and transaction results synced.

* site: Run npm audit fix.

client/asset/btc/btc.go

@@ -861,7 +861,6 @@ type intermediaryWallet struct {
 	*baseWallet
 	txFeeEstimator txFeeEstimator
 	tipRedeemer    tipRedemptionWallet
-	txLister       txLister
 
 	syncingTxHistory atomic.Bool
 }
@@ -1206,7 +1205,6 @@ func newRPCWallet(requester RawRequester, cfg *BTCCloneCFG, parsedCfg *RPCWallet
 		baseWallet:     btc,
 		txFeeEstimator: node,
 		tipRedeemer:    node,
-		txLister:       node,
 	}
 
 	w.prepareRedemptionFinder()
@@ -1397,7 +1395,6 @@ func OpenSPVWallet(cfg *BTCCloneCFG, walletConstructor BTCWalletConstructor) (*E
 			baseWallet:     btc,
 			txFeeEstimator: spvw,
 			tipRedeemer:    spvw,
-			txLister:       spvw,
 		},
 		authAddOn: &authAddOn{spvw},
 		spvNode:   spvw,
@@ -1538,6 +1535,8 @@ func (btc *baseWallet) startTxHistoryDB(ctx context.Context) (*sync.WaitGroup, e
 	btc.log.Debugf("Using tx history db at %s", dbPath)
 
 	db := NewBadgerTxDB(dbPath, btc.log)
+	btc.txHistoryDB.Store(db)
+
 	wg, err := db.Connect(ctx)
 	if err != nil {
 		return nil, err
@@ -5574,7 +5573,7 @@ func rpcTxFee(tx *ListTransactionsResult) uint64 {
 
 // idUnknownTx identifies the type and details of a transaction either made
 // or recieved by the wallet.
-func (btc *intermediaryWallet) idUnknownTx(tx *ListTransactionsResult) (*asset.WalletTransaction, error) {
+func (btc *baseWallet) idUnknownTx(tx *ListTransactionsResult) (*asset.WalletTransaction, error) {
 	txHash, err := chainhash.NewHashFromStr(tx.TxID)
 	if err != nil {
 		return nil, fmt.Errorf("error decoding tx hash %s: %v", tx.TxID, err)
@@ -5851,7 +5850,7 @@ func (btc *intermediaryWallet) idUnknownTx(tx *ListTransactionsResult) (*asset.W
 // addUnknownTransactionsToHistory checks for any transactions the wallet has
 // made or recieved that are not part of the transaction history. It scans
 // from the last point to which it had previously scanned to the current tip.
-func (btc *intermediaryWallet) addUnknownTransactionsToHistory(tip uint64) {
+func (btc *baseWallet) addUnknownTransactionsToHistory(tip uint64) {
 	txHistoryDB := btc.txDB()
 	if txHistoryDB == nil {
 		return
@@ -5869,7 +5868,7 @@ func (btc *intermediaryWallet) addUnknownTransactionsToHistory(tip uint64) {
 		blockToQuery = tip - blockQueryBuffer
 	}
 
-	txs, err := btc.txLister.listTransactionsSinceBlock(int32(blockToQuery))
+	txs, err := btc.node.listTransactionsSinceBlock(int32(blockToQuery))
 	if err != nil {
 		btc.log.Errorf("Error listing transactions since block %d: %v", blockToQuery, err)
 		return

client/asset/btc/electrum.go

@@ -5,10 +5,12 @@ package btc
 
 import (
 	"context"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"decred.org/dcrdex/client/asset"
@@ -27,10 +29,13 @@ type ExchangeWalletElectrum struct {
 
 	findRedemptionMtx   sync.RWMutex
 	findRedemptionQueue map[OutPoint]*FindRedemptionReq
+
+	syncingTxHistory atomic.Bool
 }
 
 var _ asset.Wallet = (*ExchangeWalletElectrum)(nil)
 var _ asset.Authenticator = (*ExchangeWalletElectrum)(nil)
+var _ asset.WalletHistorian = (*ExchangeWalletElectrum)(nil)
 
 // ElectrumWallet creates a new ExchangeWalletElectrum for the provided
 // configuration, which must contain the necessary details for accessing the
@@ -125,6 +130,17 @@ func (btc *ExchangeWalletElectrum) Connect(ctx context.Context) (*sync.WaitGroup
 			genesis.String(), serverFeats.Genesis)
 	}
 
+	dbWG, err := btc.startTxHistoryDB(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		dbWG.Wait()
+	}()
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -137,6 +153,15 @@ func (btc *ExchangeWalletElectrum) Connect(ctx context.Context) (*sync.WaitGroup
 		btc.monitorPeers(ctx)
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		btc.tipMtx.RLock()
+		tip := btc.currentTip
+		btc.tipMtx.RUnlock()
+		go btc.syncTxHistory(uint64(tip.Height))
+	}()
+
 	return wg, nil
 }
 
@@ -336,6 +361,8 @@ func (btc *ExchangeWalletElectrum) watchBlocks(ctx context.Context) {
 				continue
 			}
 
+			go btc.syncTxHistory(uint64(newTip.Height))
+
 			btc.log.Tracef("tip change: %d (%s) => %d (%s)", currentTip.Height, currentTip.Hash,
 				newTip.Height, newTip.Hash)
 			currentTip = newTip
@@ -347,3 +374,216 @@ func (btc *ExchangeWalletElectrum) watchBlocks(ctx context.Context) {
 		}
 	}
 }
+
+// syncTxHistory checks to see if there are any transactions which the wallet
+// has made or recieved that are not part of the transaction history, then
+// identifies and adds them. It also checks all the pending transactions to see
+// if they have been mined into a block, and if so, updates the transaction
+// history to reflect the block height.
+func (btc *ExchangeWalletElectrum) syncTxHistory(tip uint64) {
+	if !btc.syncingTxHistory.CompareAndSwap(false, true) {
+		return
+	}
+	defer btc.syncingTxHistory.Store(false)
+
+	txHistoryDB := btc.txDB()
+	if txHistoryDB == nil {
+		return
+	}
+
+	synced, _, err := btc.SyncStatus()
+	if err != nil {
+		btc.log.Errorf("Error getting sync status: %v", err)
+		return
+	}
+	if !synced {
+		return
+	}
+
+	btc.addUnknownTransactionsToHistory(tip)
+
+	pendingTxsCopy := make(map[chainhash.Hash]ExtendedWalletTx, len(btc.pendingTxs))
+	btc.pendingTxsMtx.RLock()
+	for hash, tx := range btc.pendingTxs {
+		pendingTxsCopy[hash] = tx
+	}
+	btc.pendingTxsMtx.RUnlock()
+
+	handlePendingTx := func(txHash chainhash.Hash, tx *ExtendedWalletTx) {
+		if !tx.Submitted {
+			return
+		}
+
+		gtr, err := btc.node.getWalletTransaction(&txHash)
+		if errors.Is(err, asset.CoinNotFoundError) {
+			err = txHistoryDB.RemoveTx(txHash.String())
+			if err == nil || errors.Is(err, asset.CoinNotFoundError) {
+				btc.pendingTxsMtx.Lock()
+				delete(btc.pendingTxs, txHash)
+				btc.pendingTxsMtx.Unlock()
+			} else {
+				// Leave it in the pendingPendingTxs and attempt to remove it
+				// again next time.
+				btc.log.Errorf("Error removing tx %s from the history store: %v", txHash.String(), err)
+			}
+			return
+		}
+		if err != nil {
+			btc.log.Errorf("Error getting transaction %s: %v", txHash.String(), err)
+			return
+		}
+
+		var updated bool
+		if gtr.BlockHash != "" {
+			bestHeight, err := btc.node.getBestBlockHeight()
+			if err != nil {
+				btc.log.Errorf("getBestBlockHeader: %v", err)
+				return
+			}
+			// TODO: Just get the block height with the header.
+			blockHeight := bestHeight - int32(gtr.Confirmations) + 1
+			i := 0
+			for {
+				if i > 20 || blockHeight < 0 {
+					btc.log.Errorf("Cannot find mined tx block number for %s", gtr.BlockHash)
+					return
+				}
+				bh, err := btc.ew.getBlockHeaderByHeight(btc.ctx, int64(blockHeight))
+				if err != nil {
+					btc.log.Errorf("Error getting mined tx block number %s: %v", gtr.BlockHash, err)
+					return
+				}
+				if bh.BlockHash().String() == gtr.BlockHash {
+					break
+				}
+				i++
+				blockHeight--
+			}
+			if tx.BlockNumber != uint64(blockHeight) {
+				tx.BlockNumber = uint64(blockHeight)
+				tx.Timestamp = gtr.BlockTime
+				updated = true
+			}
+		} else if gtr.BlockHash == "" && tx.BlockNumber != 0 {
+			tx.BlockNumber = 0
+			tx.Timestamp = 0
+			updated = true
+		}
+
+		var confs uint64
+		if tx.BlockNumber > 0 && tip >= tx.BlockNumber {
+			confs = tip - tx.BlockNumber + 1
+		}
+		if confs >= requiredRedeemConfirms {
+			tx.Confirmed = true
+			updated = true
+		}
+
+		if updated {
+			err = txHistoryDB.StoreTx(tx)
+			if err != nil {
+				btc.log.Errorf("Error updating tx %s: %v", txHash, err)
+				return
+			}
+
+			btc.pendingTxsMtx.Lock()
+			if tx.Confirmed {
+				delete(btc.pendingTxs, txHash)
+			} else {
+				btc.pendingTxs[txHash] = *tx
+			}
+			btc.pendingTxsMtx.Unlock()
+
+			btc.emit.TransactionNote(tx.WalletTransaction, false)
+		}
+	}
+
+	for hash, tx := range pendingTxsCopy {
+		handlePendingTx(hash, &tx)
+	}
+}
+
+// WalletTransaction returns a transaction that either the wallet has made or
+// one in which the wallet has received funds. The txID can be either a byte
+// reversed tx hash or a hex encoded coin ID.
+func (btc *ExchangeWalletElectrum) WalletTransaction(ctx context.Context, txID string) (*asset.WalletTransaction, error) {
+	coinID, err := hex.DecodeString(txID)
+	if err == nil {
+		txHash, _, err := decodeCoinID(coinID)
+		if err == nil {
+			txID = txHash.String()
+		}
+	}
+
+	txHistoryDB := btc.txDB()
+	tx, err := txHistoryDB.GetTx(txID)
+	if err != nil && !errors.Is(err, asset.CoinNotFoundError) {
+		return nil, err
+	}
+
+	if tx == nil {
+		txHash, err := chainhash.NewHashFromStr(txID)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding txid %s: %w", txID, err)
+		}
+
+		gtr, err := btc.node.getWalletTransaction(txHash)
+		if err != nil {
+			return nil, fmt.Errorf("error getting transaction %s: %w", txID, err)
+		}
+
+		var blockHeight uint32
+		if gtr.BlockHash != "" {
+			bestHeight, err := btc.node.getBestBlockHeight()
+			if err != nil {
+				return nil, fmt.Errorf("getBestBlockHeader: %v", err)
+			}
+			// TODO: Just get the block height with the header.
+			blockHeight := bestHeight - int32(gtr.Confirmations) + 1
+			i := 0
+			for {
+				if i > 20 || blockHeight < 0 {
+					return nil, fmt.Errorf("Cannot find mined tx block number for %s", gtr.BlockHash)
+				}
+				bh, err := btc.ew.getBlockHeaderByHeight(btc.ctx, int64(blockHeight))
+				if err != nil {
+					return nil, fmt.Errorf("Error getting mined tx block number %s: %v", gtr.BlockHash, err)
+				}
+				if bh.BlockHash().String() == gtr.BlockHash {
+					break
+				}
+				i++
+				blockHeight--
+			}
+		}
+
+		tx, err = btc.idUnknownTx(&ListTransactionsResult{
+			BlockHeight: blockHeight,
+			BlockTime:   gtr.BlockTime,
+			TxID:        txID,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("error identifying transaction: %v", err)
+		}
+
+		tx.BlockNumber = uint64(blockHeight)
+		tx.Timestamp = gtr.BlockTime
+		tx.Confirmed = blockHeight > 0
+		btc.addTxToHistory(tx, txHash, true, false)
+	}
+
+	return tx, nil
+}
+
+// TxHistory returns all the transactions the wallet has made. If refID is nil,
+// then transactions starting from the most recent are returned (past is ignored).
+// If past is true, the transactions prior to the refID are returned, otherwise
+// the transactions after the refID are returned. n is the number of
+// transactions to return. If n is <= 0, all the transactions will be returned.
+func (btc *ExchangeWalletElectrum) TxHistory(n int, refID *string, past bool) ([]*asset.WalletTransaction, error) {
+	txHistoryDB := btc.txDB()
+	if txHistoryDB == nil {
+		return nil, fmt.Errorf("tx database not initialized")
+	}
+	return txHistoryDB.GetTxs(n, refID, past)
+}

client/asset/btc/electrum/wallet_methods.go

@@ -38,6 +38,7 @@ const (
 	methodSignTransaction  = "signtransaction" // requires password for protected wallets
 	methodFreezeUTXO       = "freeze_utxo"
 	methodUnfreezeUTXO     = "unfreeze_utxo"
+	methodOnchainHistory   = "onchain_history"
 )
 
 // Commands gets a list of the supported wallet RPCs.
@@ -498,3 +499,21 @@ func (wc *WalletClient) GetPrivateKeys(ctx context.Context, walletPass, addr str
 	}
 	return privSplit[1], nil
 }
+
+type onchainHistoryReq struct {
+	Wallet string `json:"wallet,omitempty"`
+	From   int64  `json:"from_height,omitempty"`
+	To     int64  `json:"to_height,omitempty"`
+}
+
+func (wc *WalletClient) OnchainHistory(ctx context.Context, from, to int64) ([]TransactionResult, error) {
+	// A balance summary is included but left out here.
+	var res struct {
+		Transactions []TransactionResult `json:"transactions"`
+	}
+	err := wc.Call(ctx, methodOnchainHistory, &onchainHistoryReq{Wallet: wc.walletFile, From: from, To: to}, &res)
+	if err != nil {
+		return nil, err
+	}
+	return res.Transactions, nil
+}

client/asset/btc/electrum/wallet_types.go

@@ -69,3 +69,21 @@ type Balance struct {
 	Unconfirmed float64 // will be negative for sends
 	Immature    float64
 }
+
+// TransactionResult is is an element of the array treturned by the
+// onchainhistory RPC.
+type TransactionResult struct {
+	BcBalance          string  `json:"bc_balance"`
+	BcValue            string  `json:"bc_value"`
+	Confirmations      int64   `json:"confirmations"`
+	Date               string  `json:"date"`
+	Fee                *string `json:"fee,omitempty"`
+	FeeSat             *int64  `json:"fee_sat,omitempty"`
+	Height             int64   `json:"height"`
+	Incoming           bool    `json:"incoming"`
+	Label              string  `json:"label"`
+	MonotonicTimestamp int64   `json:"monotonic_timestamp"`
+	Timestamp          int64   `json:"timestamp"`
+	TxID               string  `json:"txid"`
+	TxposInBlock       int64   `json:"txpos_in_block"`
+}