feat(swaps): recover crashed swap deals

This commit attempts to recover swap deals that were interrupted due to
a system or `xud` crash. In the case where we are the maker and have
attempted to send payment for the second leg of the swap, we attempt to
query the swap client for the preimage of that payment in case it went
through. We can then use that preimage to try to claim the payment from
the first leg of the swap. In case the payment is known to have failed,
we simply attempt to close any open invoices and mark the swap deal as
having errored.

If an outgoing payment is still in flight and we do not have the
preimage for it, we add it to a set of "pending" swaps and check on it
on a scheduled interval until we can determine whether it has failed
or succeeded.

A new `SwapRecovery` class is introduced to contain the logic for
recovering interrupted swap deals and for tracking swaps that are
still pending. Any pending swaps are listed in the `GetInfo` response.

Raiden currently does not expose an API call to push a preimage to claim
an incoming payment or to reject an incoming payment, instead we print
a warning to the log for now.

The recovery attempts happen on `xud` startup by looking for any swap
deals in the database that have an `Active` state.

This commit includes a suite of test cases for the newly added
functionality.

Closes #1079.

lib/Xud.ts

@@ -222,7 +222,9 @@ class Xud extends EventEmitter {
     }
     if (this.grpcAPIProxy) {
       closePromises.push(this.grpcAPIProxy.close());
-      await this.grpcAPIProxy.close();
+    }
+    if (this.swaps) {
+      this.swaps.close();
     }
     await Promise.all(closePromises);
 

lib/cli/commands/getinfo.ts

@@ -4,14 +4,6 @@ import Table, { VerticalTable } from 'cli-table3';
 import colors from 'colors/safe';
 import { GetInfoRequest, GetInfoResponse, LndInfo, RaidenInfo } from '../../proto/xudrpc_pb';
 
-type generalInfo = {
-  version: string;
-  numPeers: number;
-  numPairs: number;
-  nodePubKey: string;
-  orders: {own: number, peer: number} | undefined
-};
-
 const displayChannels = (channels: any, asset: string) => {
   const table = new Table() as VerticalTable;
   Object.keys(channels).forEach((key: any) => {
@@ -74,7 +66,7 @@ const displayLndInfo = (asset: string, info: LndInfo.AsObject) => {
   }
 };
 
-const displayGeneral = (info: generalInfo) => {
+const displayGeneral = (info: GetInfoResponse.AsObject) => {
   const table = new Table() as VerticalTable;
   table.push(
     { [colors.blue('Version')]: info.version },
@@ -88,6 +80,11 @@ const displayGeneral = (info: generalInfo) => {
       { [colors.blue('Peer orders')]: info.orders.peer },
     );
   }
+  if (info.pendingSwapHashesList) {
+    table.push(
+      { [colors.blue('Pending swaps')]: JSON.stringify(info.pendingSwapHashesList) },
+    );
+  }
   console.log(colors.underline(colors.bold('\nGeneral XUD Info')));
   console.log(table.toString(), '\n');
 };
@@ -105,13 +102,7 @@ const displayRaiden = (info: RaidenInfo.AsObject) => {
 };
 
 const displayGetInfo = (response: GetInfoResponse.AsObject) => {
-  displayGeneral({
-    nodePubKey: response.nodePubKey,
-    numPairs: response.numPairs,
-    numPeers: response.numPeers,
-    version: response.version,
-    orders: response.orders,
-  });
+  displayGeneral(response);
   if (response.raiden) {
     displayRaiden(response.raiden);
   }

lib/constants/enums.ts

@@ -81,6 +81,12 @@ export enum SwapState {
   Active = 0,
   Error = 1,
   Completed = 2,
+  /**
+   * A swap that was executed but wasn't formally completed. This may occur as a result of xud
+   * crashing late in the swap process, after htlcs for both legs of the swap are set up but
+   * before the swap is formally complete.
+   */
+  Recovered = 3,
 }
 
 export enum ReputationEvent {
@@ -126,7 +132,9 @@ export enum SwapFailureReason {
   /** The swap failed due to an unrecognized error. */
   UnknownError = 12,
   /** The swap failed due to an error or unexpected behavior on behalf of the remote peer. */
-  RemoteError = 12,
+  RemoteError = 13,
+  /** The swap failed because of a system or xud crash while the swap was being executed. */
+  Crash = 14,
 }
 
 export enum DisconnectionReason {

lib/lndclient/LndClient.ts

@@ -1,6 +1,6 @@
 import grpc, { ChannelCredentials, ClientReadableStream } from 'grpc';
 import Logger from '../Logger';
-import SwapClient, { ClientStatus, SwapClientInfo } from '../swaps/SwapClient';
+import SwapClient, { ClientStatus, SwapClientInfo, PaymentState } from '../swaps/SwapClient';
 import errors from './errors';
 import { errors as swapErrors } from '../swaps/errors';
 import { LightningClient, WalletUnlockerClient } from '../proto/lndrpc_grpc_pb';
@@ -703,11 +703,12 @@ class LndClient extends SwapClient {
   }
 
   public settleInvoice = async (rHash: string, rPreimage: string) => {
+    const settleInvoiceRequest = new lndinvoices.SettleInvoiceMsg();
+    settleInvoiceRequest.setPreimage(hexToUint8Array(rPreimage));
+    await this.settleInvoiceLnd(settleInvoiceRequest);
+
     const invoiceSubscription = this.invoiceSubscriptions.get(rHash);
     if (invoiceSubscription) {
-      const settleInvoiceRequest = new lndinvoices.SettleInvoiceMsg();
-      settleInvoiceRequest.setPreimage(hexToUint8Array(rPreimage));
-      await this.settleInvoiceLnd(settleInvoiceRequest);
       this.logger.debug(`settled invoice for ${rHash}`);
       invoiceSubscription.cancel();
     }
@@ -718,12 +719,54 @@ class LndClient extends SwapClient {
     if (invoiceSubscription) {
       const cancelInvoiceRequest = new lndinvoices.CancelInvoiceMsg();
       cancelInvoiceRequest.setPaymentHash(hexToUint8Array(rHash));
-      await this.cancelInvoice(cancelInvoiceRequest);
-      this.logger.debug(`canceled invoice for ${rHash}`);
+      try {
+        await this.cancelInvoice(cancelInvoiceRequest);
+        this.logger.debug(`canceled invoice for ${rHash}`);
+      } catch (err) {
+        // handle errors due to attempting to remove an invoice that doesn't exist
+        if (err.message === 'unable to locate invoice') {
+          this.logger.debug(`attempted to cancel non-existent invoice for ${rHash}`);
+        } else if (err.message === 'invoice already canceled') {
+          this.logger.debug(`attempted to cancel already canceled invoice for ${rHash}`);
+        } else {
+          throw err;
+        }
+      }
       invoiceSubscription.cancel();
     }
   }
 
+  public lookupPayment = async (rHash: string) => {
+    const payments = await this.listPayments(true);
+    for (const payment of payments.getPaymentsList()) {
+      if (payment.getPaymentHash() === rHash) {
+        switch (payment.getStatus()) {
+          case lndrpc.Payment.PaymentStatus.SUCCEEDED:
+            const preimage = payment.getPaymentPreimage();
+            return { preimage, state: PaymentState.Succeeded };
+          case lndrpc.Payment.PaymentStatus.IN_FLIGHT:
+            return { state: PaymentState.Pending };
+          default:
+            this.logger.warn(`unexpected payment state for payment with hash ${rHash}`);
+            /* falls through */
+          case lndrpc.Payment.PaymentStatus.FAILED:
+            return { state: PaymentState.Failed };
+        }
+      }
+    }
+
+    // if no payment is found, we assume that the payment was never attempted by lnd
+    return { state: PaymentState.Failed };
+  }
+
+  private listPayments = (includeIncomplete?: boolean): Promise<lndrpc.ListPaymentsResponse> => {
+    const request = new lndrpc.ListPaymentsRequest();
+    if (includeIncomplete) {
+      request.setIncludeIncomplete(includeIncomplete);
+    }
+    return this.unaryCall<lndrpc.ListPaymentsRequest, lndrpc.ListPaymentsResponse>('listPayments', request);
+  }
+
   private addHoldInvoice = (request: lndinvoices.AddHoldInvoiceRequest): Promise<lndinvoices.AddHoldInvoiceResp> => {
     return this.unaryInvoiceCall<lndinvoices.AddHoldInvoiceRequest, lndinvoices.AddHoldInvoiceResp>('addHoldInvoice', request);
   }