Terra txm log cleanup (#5884)

* Make errors serious

* Bump terra

core/chains/terra/terratxm/txm.go

@@ -179,17 +179,20 @@ func (txm *Txm) sendMsgBatchFromAddress(gasPrice sdk.DecCoin, sender sdk.AccAddr
 	}
 	an, sn, err := tc.Account(sender)
 	if err != nil {
-		txm.lggr.Errorw("unable to read account", "err", err, "from", sender.String())
+		txm.lggr.Warnw("unable to read account", "err", err, "from", sender.String())
 		// If we can't read the account, assume transient api issues and leave msgs unstarted
 		// to retry on next poll.
 		return
 	}
 
-	txm.lggr.Debugw("simulating batch", "from", sender, "msgs", msgs)
+	txm.lggr.Debugw("simulating batch", "from", sender, "msgs", msgs, "seqnum", sn)
 	simResults, err := tc.BatchSimulateUnsigned(msgs.GetSimMsgs(), sn)
 	if err != nil {
-		txm.lggr.Errorw("unable to simulate", "err", err, "from", sender.String())
+		txm.lggr.Warnw("unable to simulate", "err", err, "from", sender.String())
 		// If we can't simulate assume transient api issue and retry on next poll.
+		// Note one rare scenario in which this can happen: the terra node misbehaves
+		// in that it confirms a txhash is present but still gives an old seq num.
+		// This is benign as the next retry will succeeds.
 		return
 	}
 	txm.lggr.Debugw("simulation results", "from", sender, "succeeded", simResults.Succeeded, "failed", simResults.Failed)
@@ -208,15 +211,16 @@ func (txm *Txm) sendMsgBatchFromAddress(gasPrice sdk.DecCoin, sender sdk.AccAddr
 	// Get the gas limit for the successful batch
 	s, err := tc.SimulateUnsigned(simResults.Succeeded.GetMsgs(), sn)
 	if err != nil {
-		// Should never happen
-		txm.lggr.Errorw("unexpected failure after successful simulation", "err", err)
+		// In the OCR context this should only happen upon stale report
+		txm.lggr.Warn("unexpected failure after successful simulation", "err", err)
 		return
 	}
 	gasLimit := s.GasInfo.GasUsed
 
 	lb, err := tc.LatestBlock()
 	if err != nil {
-		txm.lggr.Errorw("unable to get latest block", "err", err, "from", sender.String())
+		txm.lggr.Warnw("unable to get latest block", "err", err, "from", sender.String())
+		// Assume transient api issue and retry.
 		return
 	}
 	signedTx, err := tc.CreateAndSign(simResults.Succeeded.GetMsgs(), an, sn, gasLimit, txm.cfg.GasLimitMultiplier(),
@@ -241,6 +245,7 @@ func (txm *Txm) sendMsgBatchFromAddress(gasPrice sdk.DecCoin, sender sdk.AccAddr
 		resp, err = tc.Broadcast(signedTx, txtypes.BroadcastMode_BROADCAST_MODE_SYNC)
 		if err != nil {
 			// Rollback marking as broadcasted
+			// Note can happen if the node's mempool is full, where we expect errCode 20.
 			return err
 		}
 		if resp.TxResponse == nil {
@@ -249,7 +254,7 @@ func (txm *Txm) sendMsgBatchFromAddress(gasPrice sdk.DecCoin, sender sdk.AccAddr
 		}
 		if resp.TxResponse.TxHash != txHash {
 			// Should never happen
-			txm.lggr.Errorw("txhash mismatch", "got", resp.TxResponse.TxHash, "want", txHash)
+			txm.lggr.CriticalW("txhash mismatch", "got", resp.TxResponse.TxHash, "want", txHash)
 		}
 		return nil
 	})
@@ -305,6 +310,7 @@ func (txm *Txm) confirmTx(tc terraclient.Reader, txHash string, broadcasted []in
 			txm.lggr.Errorw("error looking for hash of tx, unexpected response", "tx", tx, "hash", txHash)
 			continue
 		}
+
 		txm.lggr.Infow("successfully sent batch", "hash", txHash, "msgs", broadcasted)
 		// If confirmed mark these as completed.
 		err = txm.orm.UpdateMsgsWithState(broadcasted, db.Confirmed, nil)

go.mod

@@ -54,7 +54,7 @@ require (
 	github.com/scylladb/go-reflectx v1.0.1
 	github.com/shopspring/decimal v1.3.1
 	github.com/smartcontractkit/chainlink-solana v0.2.6-0.20220125203222-e68f44abb5ce
-	github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220127174035-81f2088e26c7
+	github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220128171600-6414879ce5eb
 	github.com/smartcontractkit/helmenv v1.0.27
 	github.com/smartcontractkit/integrations-framework v1.0.42
 	github.com/smartcontractkit/libocr v0.0.0-20220125200954-5b957c834276

go.sum

@@ -2266,6 +2266,8 @@ github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220127171438-770aa33264e1
 github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220127171438-770aa33264e1/go.mod h1:Spj7zBDsK/vdJ9Yf1R0ge9WDq16UPyayERH6bX+DMOk=
 github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220127174035-81f2088e26c7 h1:k0qbvAK1U/uwegopUOS38jTBgITnDTaQPpGGz34ygD0=
 github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220127174035-81f2088e26c7/go.mod h1:AVv6Hvx6g1GY7V6iilBL8BAqBdIkW5+AYuXXXhnPM8w=
+github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220128171600-6414879ce5eb h1:QacCuRT7a7NPOazhuVTyxrIrqtTlRJjNbs+NfbvHyfU=
+github.com/smartcontractkit/chainlink-terra v0.0.5-0.20220128171600-6414879ce5eb/go.mod h1:AVv6Hvx6g1GY7V6iilBL8BAqBdIkW5+AYuXXXhnPM8w=
 github.com/smartcontractkit/helmenv v1.0.27 h1:BVxTQZQlFElh6YVK8mY9LSljX7uyx4jyImmHh/fMqEo=
 github.com/smartcontractkit/helmenv v1.0.27/go.mod h1:ef0doolSZf8ckqaWMIK2M+EPXdIKYVzttd6EXaCgCK4=
 github.com/smartcontractkit/integrations-framework v1.0.41 h1:nf/0k2fIKlhCsFlvHfhMCrfxyNHzIV7YnAc95Q7Vv50=