payments: SQL backend implementation series

config_builder.go

@@ -1228,6 +1228,23 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 
 			return nil, nil, err
 		}
+
+		// Create the payments DB.
+		//
+		// NOTE:  In the regular build, this will construct a kvdb
+		// backed payments backend. With the test_native_sql tag, it
+		// will build a SQL payments backend.
+		sqlPaymentsDB, err := d.getPaymentsStore(
+			baseDB, dbs.ChanStateDB.Backend,
+		)
+		if err != nil {
+			err = fmt.Errorf("unable to get payments store: %w",
+				err)
+
+			return nil, nil, err
+		}
+
+		dbs.PaymentsDB = sqlPaymentsDB
 	} else {
 		// Check if the invoice bucket tombstone is set. If it is, we
 		// need to return and ask the user switch back to using the
@@ -1256,40 +1273,32 @@ func (d *DefaultDatabaseBuilder) BuildDatabase(
 		if err != nil {
 			return nil, nil, err
 		}
-	}
 
-	dbs.GraphDB, err = graphdb.NewChannelGraph(graphStore, chanGraphOpts...)
-	if err != nil {
-		cleanUp()
+		// Create the payments DB.
+		kvPaymentsDB, err := paymentsdb.NewKVStore(
+			dbs.ChanStateDB,
+		)
+		if err != nil {
+			cleanUp()
 
-		err = fmt.Errorf("unable to open channel graph DB: %w", err)
-		d.logger.Error(err)
+			err = fmt.Errorf("unable to open payments DB: %w", err)
+			d.logger.Error(err)
 
-		return nil, nil, err
-	}
+			return nil, nil, err
+		}
 
-	// Mount the payments DB which is only KV for now.
-	//
-	// TODO(ziggie): Add support for SQL payments DB.
-	// Mount the payments DB for the KV store.
-	paymentsDBOptions := []paymentsdb.OptionModifier{
-		paymentsdb.WithKeepFailedPaymentAttempts(
-			cfg.KeepFailedPaymentAttempts,
-		),
+		dbs.PaymentsDB = kvPaymentsDB
 	}
-	kvPaymentsDB, err := paymentsdb.NewKVStore(
-		dbs.ChanStateDB,
-		paymentsDBOptions...,
-	)
+
+	dbs.GraphDB, err = graphdb.NewChannelGraph(graphStore, chanGraphOpts...)
 	if err != nil {
 		cleanUp()
 
-		err = fmt.Errorf("unable to open payments DB: %w", err)
+		err = fmt.Errorf("unable to open channel graph DB: %w", err)
 		d.logger.Error(err)
 
 		return nil, nil, err
 	}
-	dbs.PaymentsDB = kvPaymentsDB
 
 	// Wrap the watchtower client DB and make sure we clean up.
 	if cfg.WtClient.Active {

config_prod.go

@@ -6,6 +6,8 @@ import (
 	"context"
 
 	"github.com/lightningnetwork/lnd/kvdb"
+	paymentsdb "github.com/lightningnetwork/lnd/payments/db"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/lightningnetwork/lnd/sqldb/sqlc"
 )
 
@@ -24,3 +26,12 @@ func (d *DefaultDatabaseBuilder) getSQLMigration(ctx context.Context,
 
 	return nil, false
 }
+
+// getPaymentsStore returns a paymentsdb.DB backed by a paymentsdb.KVStore
+// implementation.
+func (d *DefaultDatabaseBuilder) getPaymentsStore(_ *sqldb.BaseDB,
+	kvBackend kvdb.Backend,
+	opts ...paymentsdb.OptionModifier) (paymentsdb.DB, error) {
+
+	return paymentsdb.NewKVStore(kvBackend, opts...)
+}

config_test_native_sql.go

@@ -4,8 +4,12 @@ package lnd
 
 import (
 	"context"
+	"database/sql"
 
 	"github.com/lightningnetwork/lnd/kvdb"
+	"github.com/lightningnetwork/lnd/lncfg"
+	paymentsdb "github.com/lightningnetwork/lnd/payments/db"
+	"github.com/lightningnetwork/lnd/sqldb"
 	"github.com/lightningnetwork/lnd/sqldb/sqlc"
 )
 
@@ -25,3 +29,28 @@ func (d *DefaultDatabaseBuilder) getSQLMigration(_ context.Context,
 		return nil, false
 	}
 }
+
+// getPaymentsStore returns a paymentsdb.DB backed by a paymentsdb.SQLStore
+// implementation.
+func (d *DefaultDatabaseBuilder) getPaymentsStore(baseDB *sqldb.BaseDB,
+	kvBackend kvdb.Backend,
+	opts ...paymentsdb.OptionModifier) (paymentsdb.DB, error) {
+
+	paymentsExecutor := sqldb.NewTransactionExecutor(
+		baseDB, func(tx *sql.Tx) paymentsdb.SQLQueries {
+			return baseDB.WithTx(tx)
+		},
+	)
+
+	queryConfig := d.cfg.DB.Sqlite.QueryConfig
+	if d.cfg.DB.Backend == lncfg.PostgresBackend {
+		queryConfig = d.cfg.DB.Postgres.QueryConfig
+	}
+
+	return paymentsdb.NewSQLStore(
+		&paymentsdb.SQLStoreConfig{
+			QueryCfg: &queryConfig,
+		},
+		paymentsExecutor, opts...,
+	)
+}

docs/release-notes/release-notes-0.21.0.md

@@ -197,6 +197,28 @@
     [4](https://github.com/lightningnetwork/lnd/pull/10542),
     [5](https://github.com/lightningnetwork/lnd/pull/10572).
 
+* Payment Store SQL implementation and migration project:
+  * Introduce an [abstract payment 
+    store](https://github.com/lightningnetwork/lnd/pull/10153) interface and
+    refacotor the payment related LND code to make it more modular.
+  * Implement the SQL backend for the [payments 
+    database](https://github.com/lightningnetwork/lnd/pull/9147)
+  * Implement query methods (QueryPayments,FetchPayment) for the [payments db 
+    SQL Backend](https://github.com/lightningnetwork/lnd/pull/10287)
+  * Implement insert methods for the [payments db 
+    SQL Backend](https://github.com/lightningnetwork/lnd/pull/10291)
+  * Implement third(final) Part of SQL backend [payment
+  functions](https://github.com/lightningnetwork/lnd/pull/10368)
+  * Finalize SQL payments implementation [enabling unit and itests
+    for SQL backend](https://github.com/lightningnetwork/lnd/pull/10292)
+  * [Thread context through payment 
+    db functions Part 1](https://github.com/lightningnetwork/lnd/pull/10307)
+  * [Thread context through payment 
+    db functions Part 2](https://github.com/lightningnetwork/lnd/pull/10308)
+  * [Finalize SQL implementation for 
+    payments db](https://github.com/lightningnetwork/lnd/pull/10373)
+
+
 ## Code Health
 
 ## Tooling and Documentation
@@ -207,8 +229,10 @@
 * Boris Nagaev
 * Elle Mouton
 * Erick Cestari
+* Gijs van Dam
 * hieblmi
 * Matt Morehouse
 * Mohamed Awnallah
 * Nishant Bansal
 * Pins
+* Ziggie

itest/lnd_payment_test.go

@@ -504,61 +504,86 @@ func testListPayments(ht *lntest.HarnessTest) {
 		expected  bool
 	}
 
-	// Create test cases to check the timestamp filters.
-	createCases := func(createTimeSeconds uint64) []testCase {
+	// Create test cases with proper rounding for start and end dates.
+	createCases := func(startTimeSeconds,
+		endTimeSeconds uint64) []testCase {
+
 		return []testCase{
 			{
 				// Use a start date same as the creation date
-				// should return us the item.
+				// (truncated) should return us the item.
 				name:      "exact start date",
-				startDate: createTimeSeconds,
+				startDate: startTimeSeconds,
 				expected:  true,
 			},
 			{
 				// Use an earlier start date should return us
 				// the item.
 				name:      "earlier start date",
-				startDate: createTimeSeconds - 1,
+				startDate: startTimeSeconds - 1,
 				expected:  true,
 			},
 			{
 				// Use a future start date should return us
 				// nothing.
 				name:      "future start date",
-				startDate: createTimeSeconds + 1,
+				startDate: startTimeSeconds + 1,
 				expected:  false,
 			},
 			{
 				// Use an end date same as the creation date
-				// should return us the item.
+				// (ceiling) should return us the item.
 				name:     "exact end date",
-				endDate:  createTimeSeconds,
+				endDate:  endTimeSeconds,
 				expected: true,
 			},
 			{
 				// Use an end date in the future should return
 				// us the item.
 				name:     "future end date",
-				endDate:  createTimeSeconds + 1,
+				endDate:  endTimeSeconds + 1,
 				expected: true,
 			},
 			{
 				// Use an earlier end date should return us
 				// nothing.
-				name:     "earlier end date",
-				endDate:  createTimeSeconds - 1,
+				name: "earlier end date",
+				// The native sql backend has a higher
+				// precision than the kv backend, the native sql
+				// backend uses microseconds, the kv backend
+				// when filtering uses seconds so we need to
+				// subtract 2 seconds to ensure the payment is
+				// not included.
+				// We could also truncate before inserting
+				// into the sql db but I rather relax this test
+				// here.
+				endDate:  endTimeSeconds - 2,
 				expected: false,
 			},
 		}
 	}
 
-	// Get the payment creation time in seconds.
-	paymentCreateSeconds := uint64(
-		p.CreationTimeNs / time.Second.Nanoseconds(),
+	// Get the payment creation time in seconds, using different approaches
+	// for start and end date comparisons to avoid rounding issues.
+	creationTime := time.Unix(0, p.CreationTimeNs)
+
+	// For start date comparisons: use truncation (floor) to include
+	// payments from the beginning of that second.
+	paymentCreateSecondsStart := uint64(
+		creationTime.Truncate(time.Second).Unix(),
+	)
+
+	// For end date comparisons: use ceiling to include payments up to the
+	// end of that second.
+	paymentCreateSecondsEnd := uint64(
+		(p.CreationTimeNs + time.Second.Nanoseconds() - 1) /
+			time.Second.Nanoseconds(),
 	)
 
 	// Create test cases from the payment creation time.
-	testCases := createCases(paymentCreateSeconds)
+	testCases := createCases(
+		paymentCreateSecondsStart, paymentCreateSecondsEnd,
+	)
 
 	// We now check the timestamp filters in `ListPayments`.
 	for _, tc := range testCases {
@@ -578,7 +603,9 @@ func testListPayments(ht *lntest.HarnessTest) {
 	}
 
 	// Create test cases from the invoice creation time.
-	testCases = createCases(uint64(invoice.CreationDate))
+	testCases = createCases(
+		uint64(invoice.CreationDate), uint64(invoice.CreationDate),
+	)
 
 	// We now do the same check for `ListInvoices`.
 	for _, tc := range testCases {

lnrpc/routerrpc/router_backend.go

@@ -1767,6 +1767,10 @@ func (r *RouterBackend) MarshallPayment(payment *paymentsdb.MPPayment) (
 		// If any of the htlcs have settled, extract a valid
 		// preimage.
 		if htlc.Settle != nil {
+			// For AMP payments all hashes will be different so we
+			// will only show the last htlc preimage, this is a
+			// current limitation for AMP payments because for
+			// MPP payments all hashes are the same.
 			preimage = htlc.Settle.Preimage
 			fee += htlc.Route.TotalFees()
 		}

lnrpc/routerrpc/router_server.go

@@ -1088,11 +1088,11 @@ func (s *Server) SendToRouteV2(ctx context.Context,
 	// db.
 	if req.SkipTempErr {
 		attempt, err = s.cfg.Router.SendToRouteSkipTempErr(
-			hash, route, firstHopRecords,
+			ctx, hash, route, firstHopRecords,
 		)
 	} else {
 		attempt, err = s.cfg.Router.SendToRoute(
-			hash, route, firstHopRecords,
+			ctx, hash, route, firstHopRecords,
 		)
 	}
 	if attempt != nil {

payments/db/errors.go

@@ -84,6 +84,12 @@ var (
 	ErrMixedBlindedAndNonBlindedPayments = errors.New("mixed blinded and " +
 		"non-blinded payments")
 
+	// ErrBlindedPaymentMissingTotalAmount is returned if we try to
+	// register a blinded payment attempt where the final hop doesn't set
+	// the total amount.
+	ErrBlindedPaymentMissingTotalAmount = errors.New("blinded payment " +
+		"final hop must set total amount")
+
 	// ErrMPPPaymentAddrMismatch is returned if we try to register an MPP
 	// shard where the payment address doesn't match existing shards.
 	ErrMPPPaymentAddrMismatch = errors.New("payment address mismatch")
@@ -136,4 +142,8 @@ var (
 	// NOTE: Only used for the kv backend.
 	ErrNoSequenceNrIndex = errors.New("payment sequence number index " +
 		"does not exist")
+
+	// errMaxPaymentsReached is used internally to signal that the maximum
+	// number of payments has been reached during a paginated query.
+	errMaxPaymentsReached = errors.New("max payments reached")
 )