Merge branch 'master' of https://github.com/hyperledger/besu into mul…

…ti-tenancy-onchain-remove-filter

.github/pull_request_template.md

@@ -6,3 +6,7 @@
 ## Fixed Issue(s)
 <!-- Please link to fixed issue(s) here using format: fixes #<issue number> -->
 <!-- Example: "fixes #2" -->
+
+## Changelog
+
+- [ ] I thought about the changelog and included a [changelog update if required](https://wiki.hyperledger.org/display/BESU/Changelog).

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Additions and Improvements
 
 * Experimental offline backup and restore has been added via the `operator x-backup-state` and `operator x-restore-state` CLI commands.  Data formats will be fluid for as long as the `x-` prefix is present in the CLI so it is advised not to rely on these backups for disaster recovery. [\#1235](https://github.com/hyperledger/besu/pull/1235)
+* Experimental ethstats support added via the `Xethstats` and `Xethstats-contact` CLI commands. [\#1239](https://github.com/hyperledger/besu/pull/1239)
 * Peers added via the JSON-RPC `admin_addPeer` and `admin_removePeer` will be shared or no longer shared via discovery respectively.  Previously they were not shared.  [\#1177](https://github.com/hyperledger/besu/pull/1177) contributed by [br0tchain](https://github.com/br0tchain).
 * New Docker Images (see below). [\#1277](https://github.com/hyperledger/besu/pull/1277)
 
@@ -21,7 +22,6 @@ The intent is that the major Java VM version or Java VM type shipped with the de
 
 #### Previously identified known issues
 
-- [Scope of logs query causing Besu to hang](KNOWN_ISSUES.md#scope-of-logs-query-causing-besu-to-hang)
 - [Logs queries missing results against chain head](KNOWN_ISSUES.md#Logs-queries-missing-results-against-chain-head)
 - [Eth/65 loses peers](KNOWN_ISSUES.md#eth65-loses-peers)
 - [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers)

KNOWN_ISSUES.md

@@ -5,12 +5,6 @@ in the current release are provided in the [Changelog](CHANGELOG.md).
 
 Known issues are open issues categorized as [Very High or High impact](https://wiki.hyperledger.org/display/BESU/Defect+Prioritisation+Policy). 
 
-## Scope of logs query causing Besu to hang
-
-[`eth_getLogs` queries that are too large or too broad can cause Besu to never return](https://github.com/hyperledger/besu/issues/944). 
-
-Workaround -> Limit the number of blocks queried by each `eth_getLogs` call.
-
 ## Logs queries missing results against chain head
 
 When using `eth_getLogs` against the head of Goerli to retrieve Eth2 deposit log events, [some results seem to be missing](https://github.com/hyperledger/besu/issues/1153). 

ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/DiscoveryPeer.java

@@ -16,10 +16,13 @@
 
 import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeer;
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL;
+import org.hyperledger.besu.ethereum.p2p.peers.Peer;
 import org.hyperledger.besu.ethereum.p2p.peers.PeerId;
 import org.hyperledger.besu.ethereum.rlp.RLPInput;
 import org.hyperledger.besu.ethereum.rlp.RLPOutput;
 
+import java.util.Optional;
+
 import org.apache.tuweni.bytes.Bytes;
 
 /**
@@ -45,6 +48,17 @@ public static DiscoveryPeer fromEnode(final EnodeURL enode) {
     return new DiscoveryPeer(enode, Endpoint.fromEnode(enode));
   }
 
+  public static Optional<DiscoveryPeer> from(final Peer peer) {
+    if (peer instanceof DiscoveryPeer) {
+      return Optional.of((DiscoveryPeer) peer);
+    }
+
+    return Optional.of(peer)
+        .map(Peer::getEnodeURL)
+        .filter(EnodeURL::isRunningDiscovery)
+        .map(DiscoveryPeer::fromEnode);
+  }
+
   public static DiscoveryPeer fromIdAndEndpoint(final Bytes id, final Endpoint endpoint) {
     return new DiscoveryPeer(endpoint.toEnode(id), endpoint);
   }

ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgent.java

@@ -27,7 +27,7 @@
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.PingPacketData;
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil;
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL;
-import org.hyperledger.besu.ethereum.p2p.peers.MaintainedPeers;
+import org.hyperledger.besu.ethereum.p2p.peers.Peer;
 import org.hyperledger.besu.ethereum.p2p.peers.PeerId;
 import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions;
 import org.hyperledger.besu.nat.NatService;
@@ -82,15 +82,13 @@ public abstract class PeerDiscoveryAgent {
   /* Is discovery enabled? */
   private boolean isActive = false;
   protected final Subscribers<PeerBondedObserver> peerBondedObservers = Subscribers.create();
-  private final MaintainedPeers maintainedPeers;
 
   protected PeerDiscoveryAgent(
       final NodeKey nodeKey,
       final DiscoveryConfiguration config,
       final PeerPermissions peerPermissions,
       final NatService natService,
-      final MetricsSystem metricsSystem,
-      final MaintainedPeers maintainedPeers) {
+      final MetricsSystem metricsSystem) {
     this.metricsSystem = metricsSystem;
     checkArgument(nodeKey != null, "nodeKey cannot be null");
     checkArgument(config != null, "provided configuration cannot be null");
@@ -104,7 +102,7 @@ protected PeerDiscoveryAgent(
 
     this.config = config;
     this.nodeKey = nodeKey;
-    this.maintainedPeers = maintainedPeers;
+
     id = nodeKey.getPublicKey().getEncodedBytes();
   }
 
@@ -176,7 +174,6 @@ private PeerDiscoveryController createController(final DiscoveryPeer localNode)
         .peerPermissions(peerPermissions)
         .peerBondedObservers(peerBondedObservers)
         .metricsSystem(metricsSystem)
-        .maintainedPeers(maintainedPeers)
         .build();
   }
 
@@ -311,7 +308,10 @@ public boolean isActive() {
     return isActive;
   }
 
-  public void bond(final DiscoveryPeer peer) {
-    controller.ifPresent(c -> c.handleBondingRequest(peer));
+  public void bond(final Peer peer) {
+    controller.ifPresent(
+        c -> {
+          DiscoveryPeer.from(peer).ifPresent(c::handleBondingRequest);
+        });
   }
 }

ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/VertxPeerDiscoveryAgent.java

@@ -23,7 +23,6 @@
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.PeerDiscoveryController.AsyncExecutor;
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.TimerUtil;
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.VertxTimerUtil;
-import org.hyperledger.besu.ethereum.p2p.peers.MaintainedPeers;
 import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions;
 import org.hyperledger.besu.metrics.BesuMetricCategory;
 import org.hyperledger.besu.nat.NatService;
@@ -63,9 +62,8 @@ public VertxPeerDiscoveryAgent(
       final DiscoveryConfiguration config,
       final PeerPermissions peerPermissions,
       final NatService natService,
-      final MetricsSystem metricsSystem,
-      final MaintainedPeers maintainedPeers) {
-    super(nodeKey, config, peerPermissions, natService, metricsSystem, maintainedPeers);
+      final MetricsSystem metricsSystem) {
+    super(nodeKey, config, peerPermissions, natService, metricsSystem);
     checkArgument(vertx != null, "vertx instance cannot be null");
     this.vertx = vertx;
 

ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/discovery/internal/PeerDiscoveryController.java

@@ -25,7 +25,6 @@
 import org.hyperledger.besu.ethereum.p2p.discovery.PeerBondedObserver;
 import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryEvent;
 import org.hyperledger.besu.ethereum.p2p.discovery.PeerDiscoveryStatus;
-import org.hyperledger.besu.ethereum.p2p.peers.MaintainedPeers;
 import org.hyperledger.besu.ethereum.p2p.peers.Peer;
 import org.hyperledger.besu.ethereum.p2p.peers.PeerId;
 import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions;
@@ -159,8 +158,7 @@ private PeerDiscoveryController(
       final PeerRequirement peerRequirement,
       final PeerPermissions peerPermissions,
       final Subscribers<PeerBondedObserver> peerBondedObservers,
-      final MetricsSystem metricsSystem,
-      final MaintainedPeers maintainedPeers) {
+      final MetricsSystem metricsSystem) {
     this.timerUtil = timerUtil;
     this.nodeKey = nodeKey;
     this.localPeer = localPeer;
@@ -176,10 +174,6 @@ private PeerDiscoveryController(
 
     this.peerPermissions = new PeerDiscoveryPermissions(localPeer, peerPermissions);
 
-    // Listening to the added and removed peer event to update the maintainted peers
-    maintainedPeers.subscribeAdd(this::onPeerAdded);
-    maintainedPeers.subscribeRemove(this::onPeerRemoved);
-
     metricsSystem.createIntegerGauge(
         BesuMetricCategory.NETWORK,
         "discovery_inflight_interactions_current",
@@ -201,18 +195,6 @@ private PeerDiscoveryController(
             "type");
   }
 
-  private void onPeerAdded(final Peer peer, final boolean wasAdded) {
-    if (wasAdded) {
-      addToPeerTable(DiscoveryPeer.fromEnode(peer.getEnodeURL()));
-    }
-  }
-
-  private void onPeerRemoved(final Peer peer, final boolean wasRemoved) {
-    if (wasRemoved) {
-      peerTable.tryEvict(peer);
-    }
-  }
-
   public static Builder builder() {
     return new Builder();
   }
@@ -712,7 +694,6 @@ public static class Builder {
     private final List<DiscoveryPeer> bootstrapNodes = new ArrayList<>();
     private PeerTable peerTable;
     private Subscribers<PeerBondedObserver> peerBondedObservers = Subscribers.create();
-    private MaintainedPeers maintainedPeers = new MaintainedPeers();
 
     // Required dependencies
     private NodeKey nodeKey;
@@ -743,8 +724,7 @@ public PeerDiscoveryController build() {
           peerRequirement,
           peerPermissions,
           peerBondedObservers,
-          metricsSystem,
-          maintainedPeers);
+          metricsSystem);
     }
 
     private void validate() {
@@ -836,11 +816,5 @@ public Builder metricsSystem(final MetricsSystem metricsSystem) {
       this.metricsSystem = metricsSystem;
       return this;
     }
-
-    public Builder maintainedPeers(final MaintainedPeers maintainedPeers) {
-      checkNotNull(maintainedPeers);
-      this.maintainedPeers = maintainedPeers;
-      return this;
-    }
   }
 }

ethereum/p2p/src/main/java/org/hyperledger/besu/ethereum/p2p/network/DefaultP2PNetwork.java

@@ -265,6 +265,7 @@ public void awaitStop() {
   @Override
   public boolean addMaintainConnectionPeer(final Peer peer) {
     final boolean wasAdded = maintainedPeers.add(peer);
+    peerDiscoveryAgent.bond(peer);
     rlpxAgent.connect(peer);
     return wasAdded;
   }
@@ -443,13 +444,7 @@ private void validate() {
     private PeerDiscoveryAgent createDiscoveryAgent() {
 
       return new VertxPeerDiscoveryAgent(
-          vertx,
-          nodeKey,
-          config.getDiscovery(),
-          peerPermissions,
-          natService,
-          metricsSystem,
-          maintainedPeers);
+          vertx, nodeKey, config.getDiscovery(), peerPermissions, natService, metricsSystem);
     }
 
     private RlpxAgent createRlpxAgent(

ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryAgentTest.java

@@ -31,6 +31,7 @@
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.NeighborsPacketData;
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.Packet;
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.PacketType;
+import org.hyperledger.besu.ethereum.p2p.peers.DefaultPeer;
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL;
 import org.hyperledger.besu.ethereum.p2p.peers.Peer;
 import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions;
@@ -210,6 +211,115 @@ public void peerTable_disallowPeer() {
     assertThat(agent.streamDiscoveredPeers()).hasSize(0);
   }
 
+  @Test
+  public void bond_supplyGenericPeer() {
+    final MockPeerDiscoveryAgent otherNode = helper.startDiscoveryAgent();
+    assertThat(otherNode.getAdvertisedPeer().isPresent()).isTrue();
+    final DiscoveryPeer remotePeer = otherNode.getAdvertisedPeer().get();
+    final Peer genericPeer = DefaultPeer.fromEnodeURL(remotePeer.getEnodeURL());
+
+    final PeerPermissions peerPermissions = mock(PeerPermissions.class);
+    final MockPeerDiscoveryAgent agent =
+        helper.createDiscoveryAgent(helper.agentBuilder().peerPermissions(peerPermissions));
+
+    when(peerPermissions.isPermitted(any(), any(), any())).thenReturn(true);
+
+    // Start agent and bond
+    assertThat(agent.start(30303)).isCompleted();
+    assertThat(agent.streamDiscoveredPeers()).isEmpty();
+    agent.bond(genericPeer);
+
+    // We should send an outgoing ping
+    List<IncomingPacket> remoteIncomingPackets = otherNode.getIncomingPackets();
+    assertThat(remoteIncomingPackets).hasSize(2);
+    final IncomingPacket firstMsg = remoteIncomingPackets.get(0);
+    assertThat(firstMsg.packet.getType()).isEqualTo(PacketType.PING);
+    // The remote peer will send a PING and we'll respond with a return PONG
+    assertThat(firstMsg.fromAgent).isEqualTo(agent);
+    final IncomingPacket secondMsg = remoteIncomingPackets.get(1);
+    assertThat(secondMsg.packet.getType()).isEqualTo(PacketType.PONG);
+    assertThat(secondMsg.fromAgent).isEqualTo(agent);
+
+    // The peer should now be bonded
+    assertThat(agent.streamDiscoveredPeers()).contains(remotePeer);
+  }
+
+  @Test
+  public void bond_allowOutgoingBonding() {
+    final MockPeerDiscoveryAgent otherNode = helper.startDiscoveryAgent();
+    assertThat(otherNode.getAdvertisedPeer().isPresent()).isTrue();
+    final DiscoveryPeer remotePeer = otherNode.getAdvertisedPeer().get();
+
+    final PeerPermissions peerPermissions = mock(PeerPermissions.class);
+    final MockPeerDiscoveryAgent agent =
+        helper.createDiscoveryAgent(helper.agentBuilder().peerPermissions(peerPermissions));
+
+    when(peerPermissions.isPermitted(any(), any(), any())).thenReturn(false);
+    when(peerPermissions.isPermitted(
+            any(), eq(remotePeer), eq(Action.DISCOVERY_ALLOW_OUTBOUND_BONDING)))
+        .thenReturn(true);
+
+    // Start agent and bond
+    assertThat(agent.start(30303)).isCompleted();
+    agent.bond(remotePeer);
+
+    // We should send an outgoing ping
+    List<IncomingPacket> remoteIncomingPackets = otherNode.getIncomingPackets();
+    assertThat(remoteIncomingPackets).hasSize(1);
+    final IncomingPacket firstMsg = remoteIncomingPackets.get(0);
+    assertThat(firstMsg.packet.getType()).isEqualTo(PacketType.PING);
+    assertThat(firstMsg.fromAgent).isEqualTo(agent);
+  }
+
+  @Test
+  public void bond_peerWithDiscoveryDisabled() {
+    // Create a peer with discovery disabled
+    final MockPeerDiscoveryAgent otherNode = helper.startDiscoveryAgent();
+    assertThat(otherNode.getAdvertisedPeer().isPresent()).isTrue();
+    final DiscoveryPeer remotePeer = otherNode.getAdvertisedPeer().get();
+    final EnodeURL enodeWithDiscoveryDisabled =
+        EnodeURL.builder().configureFromEnode(remotePeer.getEnodeURL()).disableDiscovery().build();
+    final Peer peerWithDisabledDiscovery = DefaultPeer.fromEnodeURL(enodeWithDiscoveryDisabled);
+
+    final PeerPermissions peerPermissions = mock(PeerPermissions.class);
+    final MockPeerDiscoveryAgent agent =
+        helper.createDiscoveryAgent(helper.agentBuilder().peerPermissions(peerPermissions));
+
+    when(peerPermissions.isPermitted(any(), any(), any())).thenReturn(true);
+
+    // Start agent and bond
+    assertThat(agent.start(30303)).isCompleted();
+    agent.bond(peerWithDisabledDiscovery);
+
+    // We should not send any messages to peer with discovery disabled
+    List<IncomingPacket> remoteIncomingPackets = otherNode.getIncomingPackets();
+    assertThat(remoteIncomingPackets).hasSize(0);
+  }
+
+  @Test
+  public void bond_disallowOutgoingBonding() {
+    final MockPeerDiscoveryAgent otherNode = helper.startDiscoveryAgent();
+    assertThat(otherNode.getAdvertisedPeer().isPresent()).isTrue();
+    final DiscoveryPeer remotePeer = otherNode.getAdvertisedPeer().get();
+
+    final PeerPermissions peerPermissions = mock(PeerPermissions.class);
+    final MockPeerDiscoveryAgent agent =
+        helper.createDiscoveryAgent(helper.agentBuilder().peerPermissions(peerPermissions));
+
+    when(peerPermissions.isPermitted(any(), any(), any())).thenReturn(true);
+    when(peerPermissions.isPermitted(
+            any(), eq(remotePeer), eq(Action.DISCOVERY_ALLOW_OUTBOUND_BONDING)))
+        .thenReturn(false);
+
+    // Start agent and bond
+    assertThat(agent.start(30303)).isCompleted();
+    agent.bond(remotePeer);
+
+    // We should not send an outgoing ping
+    List<IncomingPacket> remoteIncomingPackets = otherNode.getIncomingPackets();
+    assertThat(remoteIncomingPackets).hasSize(0);
+  }
+
   @Test
   public void bonding_allowIncomingBonding() {
     // Start an agent with no bootstrap peers.

ethereum/p2p/src/test/java/org/hyperledger/besu/ethereum/p2p/discovery/PeerDiscoveryTestHelper.java

@@ -27,7 +27,6 @@
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.PingPacketData;
 import org.hyperledger.besu.ethereum.p2p.discovery.internal.PongPacketData;
 import org.hyperledger.besu.ethereum.p2p.peers.EnodeURL;
-import org.hyperledger.besu.ethereum.p2p.peers.MaintainedPeers;
 import org.hyperledger.besu.ethereum.p2p.peers.Peer;
 import org.hyperledger.besu.ethereum.p2p.permissions.PeerPermissions;
 import org.hyperledger.besu.nat.NatService;
@@ -265,10 +264,8 @@ public MockPeerDiscoveryAgent build() {
       config.setAdvertisedHost(advertisedHost);
       config.setBindPort(port);
       config.setActive(active);
-      MaintainedPeers maintainedPeers = new MaintainedPeers();
 
-      return new MockPeerDiscoveryAgent(
-          nodeKey, config, peerPermissions, agents, natService, maintainedPeers);
+      return new MockPeerDiscoveryAgent(nodeKey, config, peerPermissions, agents, natService);
     }
   }
 }