Merge branch 'lightningdevkit:main' into issue438

Author: mattfaltyn

.github/workflows/build.yml

@@ -275,6 +275,27 @@ jobs:
       - name: For each commit, run cargo check (including in fuzz)
         run: ci/check-each-commit.sh upstream/main
 
+  check_release:
+    runs-on: ubuntu-latest
+    env:
+      TOOLCHAIN: stable
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.TOOLCHAIN }}
+          override: true
+          profile: minimal
+      - name: Run cargo check for release build.
+        run: |
+          cargo check --release
+          cargo check --no-default-features --features=no-std --release
+          cargo doc --release
+
   fuzz:
     runs-on: ubuntu-latest
     env:

CHANGELOG.md

@@ -1,3 +1,163 @@
+# 0.0.108 - 2022-06-10
+
+## Bug Fixes
+ * Fixed `lightning-background-processor` build in release mode.
+
+In total, this release features 9 files changed, 120 insertions, 74
+deletions in 5 commits from 4 authors, in alphabetical order:
+ * Elias Rohrer
+ * Matt Corallo
+ * Max Fang
+ * Viktor Tigerström
+
+# 0.0.107 - 2022-06-08
+
+## API Updates
+ * Channels larger than 16777215 sats (Wumbo!) are now supported and can be
+   enabled for inbound channels using
+   `ChannelHandshakeLimits::max_funding_satoshis` (#1425).
+ * Support for feature `option_zeroconf`, allowing immediate forwarding of
+   payments after channel opening. This is configured for outbound channels
+   using `ChannelHandshakeLimits::trust_own_funding_0conf` whereas
+   `ChannelManager::accept_inbound_channel_from_trusted_peer_0conf` has to be
+   used for accepting inbound channels (#1401, #1505).
+ * `ChannelManager::claim_funds` no longer returns a `bool` to indicate success.
+   Instead, an `Event::PaymentClaimed` is generated if the claim was successful.
+   Likewise, `ChannelManager::fail_htlc_backwards` no longer has a return value
+   (#1434).
+ * `lightning-rapid-gossip-sync` is a new crate for syncing gossip data from a
+   server, primarily aimed at mobile devices (#1155).
+ * `RapidGossipSync` can be passed to `BackgroundProcessor` in order to persist
+   the `NetworkGraph` and handle `NetworkUpdate`s during event handling (#1433,
+   #1517).
+ * `NetGraphMsgHandler` has been renamed to `P2PGossipSync`, the `network_graph`
+    module has been renamed to `gossip`, and `NetworkUpdate::ChannelClosed` has
+   been renamed `NetworkUpdate::ChannelFailure` (#1159).
+ * Added a `filtered_block_connected` method to `chain::Listen` and a default
+   implementation of `block_connected` for those fetching filtered instead of
+   full blocks (#1453).
+ * The `lightning-block-sync` crate's `BlockSource` trait methods now take
+   `&self` instead of `&mut self` (#1307).
+ * `inbound_payment` module is now public to allow for creating invoices without
+   a `ChannelManager` (#1384).
+ * `lightning-block-sync`'s `init` and `poll` modules support `&dyn BlockSource`
+   which can be determined at runtime (#1423).
+ * `lightning-invoice` crate's `utils` now accept an expiration time (#1422,
+   #1474).
+ * `Event::PaymentForwarded` includes `prev_channel_id` and `next_channel_id`
+   (#1419, #1475).
+ * `chain::Watch::release_pending_monitor_events`' return type now associates
+   `MonitorEvent`s with funding `OutPoints` (#1475).
+ * `lightning-background-processor` crate's `Persister` trait has been moved to
+   `lightning` crate's `util::persist` module, which now has a general
+   `KVStorePersister` trait. Blanket implementations of `Persister` and
+   `chainmonitor::Persist` are given for types implementing `KVStorePersister`.
+   ` lightning-persister`'s `FilesystemPersister` implements `KVStorePersister`
+   (#1417).
+ * `ChannelDetails` and `ChannelCounterparty` include fields for HTLC minimum
+   and maximum values (#1378).
+ * Added a `max_inbound_htlc_value_in_flight_percent_of_channel` field to
+   `ChannelHandshakeConfig`, capping the total value of outstanding inbound
+   HTLCs for a channel (#1444).
+ * `ProbabilisticScorer` is parameterized by a `Logger`, which it uses to log
+   channel liquidity updates or lack thereof (#1405).
+ * `ChannelDetails` has an `outbound_htlc_limit_msat` field, which should be
+   used in routing instead of `outbound_capacity_msat` (#1435).
+ * `ProbabilisticScorer`'s channel liquidities can be logged via
+   `debug_log_liquidity_stats` (#1460).
+ * `BackgroundProcessor` now takes an optional `WriteableScore` which it will
+   persist using the `Persister` trait's new `persist_scorer` method (#1416).
+ * Upgraded to `bitcoin` crate version 0.28.1 (#1389).
+ * `ShutdownScript::new_witness_program` now takes a `WitnessVersion` instead of
+   a `NonZeroU8` (#1389).
+ * Channels will no longer be automatically force closed when the counterparty
+   is disconnected due to incompatibility (#1429).
+ * `ChannelManager` methods for funding, accepting, and closing channels now
+   take a `counterparty_node_id` parameter, which has also been added as a field
+   to `Event::FundingGenerationReady` (#1479, #1485).
+ * `InvoicePayer::new` now takes a `Retry` enum (replacing the `RetryAttempts`
+   struct), which supports both attempt- and timeout-based retrying (#1418).
+ * `Score::channel_penalty_msat` takes a `ChannelUsage` struct, which contains
+   the capacity as an `EffectiveCapacity` enum and any potential in-flight HTLC
+   value, rather than a single `u64`. Used by `ProbabilisticScorer` for more
+   accurate penalties (#1456).
+ * `build_route_from_hops` is a new function useful for constructing a `Route`
+   given a specific list of public keys (#1491).
+ * `FundingLocked` message has been renamed `ChannelReady`, and related
+   identifiers have been renamed accordingly (#1506).
+ * `core2::io` or `std::io` (depending on feature flags `no-std` or `std`) is
+   exported as a `lightning::io` module (#1504).
+ * The deprecated `Scorer` has been removed in favor or `ProbabilisticScorer`
+   (#1512).
+
+## Performance Improvements
+ * `lightning-persister` crate's `FilesystemPersister` is faster by 15x (#1404).
+ * Log gossip query messages at `GOSSIP` instead of `TRACE` to avoid
+   overwhelming default logging (#1421).
+ * `PeerManager` supports processing messages from different peers in parallel,
+   and this is taken advantage of in gossip processing (#1023).
+ * Greatly reduced per-channel and per-node memory usage due to upgrade of
+   `secp256k1` crate to 0.22.1 and `bitcoin` crate to 0.28.1
+ * Reduced per-peer memory usage in `PeerManager` (#1472).
+
+## Spec Compliance
+ * `find_route` now assumes variable-length onions by default for nodes where
+   support for the feature is unknown (#1414).
+ * A `warn` message is now sent when receiving a `channel_reestablish` with an
+   old commitment transaction number rather than immediately force-closing the
+   channel (#1430).
+ * When a `channel_update` message is included in an onion error's `failuremsg`,
+   its message type is now encoded. Reading such messages is also supported
+   (#1465).
+
+## Bug Fixes
+ * Fixed a bug where crashing while persisting a `ChannelMonitorUpdate` for a
+   part of a multi-path payment could cause loss of funds due to a partial
+   payment claim on restart (#1434).
+ * `BackgroundProcessor` has been fixed to improve serialization reliability on
+   slow systems which can avoid force-closes (#1436).
+ * `gossip_timestamp_filter` filters are now honored when sending gossip to
+   peers (#1452).
+ * During a reorg, only force-close a channel if its funding transaction is
+   unconfirmed rather than as it loses confirmations (#1461).
+ * Fixed a rare panic in `lightning-net-tokio` when fetching a peer's socket
+   address after the connection has been closed caused by a race condition
+   (#1449).
+ * `find_route` will no longer return routes that would cause onion construction
+   to fail in some cases (#1476).
+ * `ProbabilisticScorer` uses more precision when approximating `log10` (#1406).
+
+## Serialization Compatibility
+ * All above new events/fields are ignored by prior clients. All above new
+   events/fields are not present when reading objects serialized by prior
+   versions of the library.
+ * `ChannelManager` serialization is no longer compatible with versions prior to
+   0.0.99 (#1401).
+ * Channels with `option_zeroconf` feature enabled (not required for 0-conf
+   channel use) will be unreadable by versions prior to 0.0.107 (#1401, #1505).
+
+In total, this release features 96 files changed, 9304 insertions, 4503
+deletions in 153 commits from 18 authors, in alphabetical order:
+ * Arik Sosman
+ * Devrandom
+ * Duncan Dean
+ * Elias Rohrer
+ * Jeffrey Czyz
+ * John Cantrell
+ * John Corser
+ * Jurvis Tan
+ * Justin Moon
+ * KaFai Choi
+ * Mateusz Faltyn
+ * Matt Corallo
+ * Valentine Wallace
+ * Viktor Tigerström
+ * Vincenzo Palazzo
+ * atalw
+ * dependabot[bot]
+ * shamardy
+
+
 # 0.0.106 - 2022-04-03
 
 ## API Updates

README.md

@@ -50,6 +50,8 @@ Crates
   this is a simple alternative to implementing the required network stack, especially for those already using Tokio.
 6. [lightning-persister](./lightning-persister)
   Utilities to manage Rust-Lightning channel data persistence and retrieval.
+7. [lightning-rapid-gossip-sync](./lightning-rapid-gossip-sync)
+  Client for rapid gossip graph syncing, aimed primarily at mobile clients.
 
 About
 -----------

fuzz/src/full_stack.rs

@@ -163,7 +163,7 @@ type ChannelMan = ChannelManager<
 	EnforcingSigner,
 	Arc<chainmonitor::ChainMonitor<EnforcingSigner, Arc<dyn chain::Filter>, Arc<TestBroadcaster>, Arc<FuzzEstimator>, Arc<dyn Logger>, Arc<TestPersister>>>,
 	Arc<TestBroadcaster>, Arc<KeyProvider>, Arc<FuzzEstimator>, Arc<dyn Logger>>;
-type PeerMan<'a> = PeerManager<Peer<'a>, Arc<ChannelMan>, Arc<P2PGossipSync<Arc<NetworkGraph>, Arc<dyn chain::Access>, Arc<dyn Logger>>>, Arc<dyn Logger>, IgnoringMessageHandler>;
+type PeerMan<'a> = PeerManager<Peer<'a>, Arc<ChannelMan>, Arc<P2PGossipSync<Arc<NetworkGraph<Arc<dyn Logger>>>, Arc<dyn chain::Access>, Arc<dyn Logger>>>, Arc<dyn Logger>, IgnoringMessageHandler>;
 
 struct MoneyLossDetector<'a> {
 	manager: Arc<ChannelMan>,
@@ -395,7 +395,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
 	// it's easier to just increment the counter here so the keys don't change.
 	keys_manager.counter.fetch_sub(1, Ordering::AcqRel);
 	let our_id = PublicKey::from_secret_key(&Secp256k1::signing_only(), &keys_manager.get_node_secret(Recipient::Node).unwrap());
-	let network_graph = Arc::new(NetworkGraph::new(genesis_block(network).block_hash()));
+	let network_graph = Arc::new(NetworkGraph::new(genesis_block(network).block_hash(), Arc::clone(&logger)));
 	let gossip_sync = Arc::new(P2PGossipSync::new(Arc::clone(&network_graph), None, Arc::clone(&logger)));
 	let scorer = FixedPenaltyScorer::with_penalty(0);
 
@@ -460,7 +460,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
 					final_cltv_expiry_delta: 42,
 				};
 				let random_seed_bytes: [u8; 32] = keys_manager.get_secure_random_bytes();
-				let route = match find_route(&our_id, &params, &network_graph, None, Arc::clone(&logger), &scorer, &random_seed_bytes) {
+				let route = match find_route(&our_id, &params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &random_seed_bytes) {
 					Ok(route) => route,
 					Err(_) => return,
 				};
@@ -484,7 +484,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
 					final_cltv_expiry_delta: 42,
 				};
 				let random_seed_bytes: [u8; 32] = keys_manager.get_secure_random_bytes();
-				let mut route = match find_route(&our_id, &params, &network_graph, None, Arc::clone(&logger), &scorer, &random_seed_bytes) {
+				let mut route = match find_route(&our_id, &params, &network_graph.read_only(), None, Arc::clone(&logger), &scorer, &random_seed_bytes) {
 					Ok(route) => route,
 					Err(_) => return,
 				};

fuzz/src/process_network_graph.rs

@@ -1,22 +1,27 @@
 // Imports that need to be added manually
+use lightning::util::logger::Logger;
 use lightning_rapid_gossip_sync::RapidGossipSync;
+
 use utils::test_logger;
 
+use std::sync::Arc;
+
 /// Actual fuzz test, method signature and name are fixed
-fn do_test(data: &[u8]) {
+fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
 	let block_hash = bitcoin::BlockHash::default();
-	let network_graph = lightning::routing::gossip::NetworkGraph::new(block_hash);
+	let logger: Arc<dyn Logger> = Arc::new(test_logger::TestLogger::new("".to_owned(), out));
+	let network_graph = lightning::routing::gossip::NetworkGraph::new(block_hash, logger);
 	let rapid_sync = RapidGossipSync::new(&network_graph);
 	let _ = rapid_sync.update_network_graph(data);
 }
 
 /// Method that needs to be added manually, {name}_test
-pub fn process_network_graph_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-	do_test(data);
+pub fn process_network_graph_test<Out: test_logger::Output>(data: &[u8], out: Out) {
+	do_test(data, out);
 }
 
 /// Method that needs to be added manually, {name}_run
 #[no_mangle]
 pub extern "C" fn process_network_graph_run(data: *const u8, datalen: usize) {
-	do_test(unsafe { std::slice::from_raw_parts(data, datalen) });
+	do_test(unsafe { std::slice::from_raw_parts(data, datalen) }, test_logger::DevNull {});
 }

fuzz/src/router.rs

@@ -162,7 +162,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
 	let logger: Arc<dyn Logger> = Arc::new(test_logger::TestLogger::new("".to_owned(), out));
 
 	let our_pubkey = get_pubkey!();
-	let net_graph = NetworkGraph::new(genesis_block(Network::Bitcoin).header.block_hash());
+	let net_graph = NetworkGraph::new(genesis_block(Network::Bitcoin).header.block_hash(), Arc::clone(&logger));
 
 	let mut node_pks = HashSet::new();
 	let mut scid = 42;
@@ -267,7 +267,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
 						final_value_msat: slice_to_be64(get_slice!(8)),
 						final_cltv_expiry_delta: slice_to_be32(get_slice!(4)),
 					};
-					let _ = find_route(&our_pubkey, &route_params, &net_graph,
+					let _ = find_route(&our_pubkey, &route_params, &net_graph.read_only(),
 						first_hops.map(|c| c.iter().collect::<Vec<_>>()).as_ref().map(|a| a.as_slice()),
 						Arc::clone(&logger), &scorer, &random_seed_bytes);
 				}

lightning-background-processor/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lightning-background-processor"
-version = "0.0.106"
+version = "0.0.108"
 authors = ["Valentine Wallace <vwallace@protonmail.com>"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -15,10 +15,10 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
 bitcoin = "0.28.1"
-lightning = { version = "0.0.106", path = "../lightning", features = ["std"] }
-lightning-rapid-gossip-sync = { version = "0.0.106", path = "../lightning-rapid-gossip-sync" }
+lightning = { version = "0.0.108", path = "../lightning", features = ["std"] }
+lightning-rapid-gossip-sync = { version = "0.0.108", path = "../lightning-rapid-gossip-sync" }
 
 [dev-dependencies]
-lightning = { version = "0.0.106", path = "../lightning", features = ["_test_utils"] }
-lightning-invoice = { version = "0.14.0", path = "../lightning-invoice" }
-lightning-persister = { version = "0.0.106", path = "../lightning-persister" }
+lightning = { version = "0.0.108", path = "../lightning", features = ["_test_utils"] }
+lightning-invoice = { version = "0.16.0", path = "../lightning-invoice" }
+lightning-persister = { version = "0.0.108", path = "../lightning-persister" }