Merge pull request #554 from NethermindEth/traceblock

various fixes to JSON RPC and JSON RPC testing

.travis.yml

@@ -26,12 +26,6 @@ jobs:
     - stage: run tests
       script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind/Ethereum.Basic.Test
       name: "Ethereum.Basic.Test"
-    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind.DataMarketplace.Consumers.Test
-      name: "Nethermind.DataMarketplace.Consumers.Test"
-    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind.DataMarketplace.Integration.Test
-      name: "Nethermind.DataMarketplace.Integration.Test"  
-    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind.DataMarketplace.Test
-      name: "Nethermind.DataMarketplace.Test"  
     - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind/Ethereum.Blockchain.Block.Test
       name: "Ethereum.Blockchain.Block.Test"
     - script: dotnet test src/Nethermind/Ethereum.Blockchain.Test
@@ -94,4 +88,10 @@ jobs:
     - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*%2c[Nethermind.Core.Test]*" src/Nethermind/Nethermind.Store.Test
       name: "Nethermind.Store.Test"
     - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*%2c[Nethermind.Core.Test]*" src/Nethermind/Nethermind.Wallet.Test
-      name: "Nethermind.Wallet.Test"
+      name: "Nethermind.Wallet.Test"
+    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind/Nethermind.DataMarketplace.Consumers.Test
+      name: "Nethermind.DataMarketplace.Consumers.Test"
+    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind/Nethermind.DataMarketplace.Integration.Test
+      name: "Nethermind.DataMarketplace.Integration.Test"  
+    - script: dotnet test /p:CollectCoverage=true /p:CoverletOutputFormat=opencover /p:Exclude="[Nethermind.HashLib]*" src/Nethermind/Nethermind.DataMarketplace.Test
+      name: "Nethermind.DataMarketplace.Test"  

docs/source/cli.rst

@@ -115,6 +115,13 @@ personal
 
  - personal.unlockAccount(addressHex, password) - 
 
+system
+^^^^^^
+
+ - system.getVariable(name, defaultValue) - 
+
+ - system.memory - 
+
 web3
 ^^^^
 

docs/source/configuration.rst

@@ -35,6 +35,54 @@ DbConfig
 
  CodeDbWriteBufferSize
 
+ ConfigsDbBlockCacheSize
+
+ ConfigsDbCacheIndexAndFilterBlocks
+
+ ConfigsDbWriteBufferNumber
+
+ ConfigsDbWriteBufferSize
+
+ ConsumerDepositApprovalsDbBlockCacheSize
+
+ ConsumerDepositApprovalsDbCacheIndexAndFilterBlocks
+
+ ConsumerDepositApprovalsDbWriteBufferNumber
+
+ ConsumerDepositApprovalsDbWriteBufferSize
+
+ ConsumerReceiptsDbBlockCacheSize
+
+ ConsumerReceiptsDbCacheIndexAndFilterBlocks
+
+ ConsumerReceiptsDbWriteBufferNumber
+
+ ConsumerReceiptsDbWriteBufferSize
+
+ ConsumerSessionsDbBlockCacheSize
+
+ ConsumerSessionsDbCacheIndexAndFilterBlocks
+
+ ConsumerSessionsDbWriteBufferNumber
+
+ ConsumerSessionsDbWriteBufferSize
+
+ DepositsDbBlockCacheSize
+
+ DepositsDbCacheIndexAndFilterBlocks
+
+ DepositsDbWriteBufferNumber
+
+ DepositsDbWriteBufferSize
+
+ EthRequestsDbBlockCacheSize
+
+ EthRequestsDbCacheIndexAndFilterBlocks
+
+ EthRequestsDbWriteBufferNumber
+
+ EthRequestsDbWriteBufferSize
+
  HeadersDbBlockCacheSize
 
  HeadersDbCacheIndexAndFilterBlocks
@@ -148,15 +196,15 @@ These items need only be set when testing with Hive (Ethereum Foundation tool)
 
  BlocksDir
    Path to a directory with additional blocks.
-   default value: null
+   default value: "/blocks"
 
  ChainFile
    Path to a file with a test chain definition.
-   default value: null
+   default value: "/chain.rlp"
 
  KeysDir
    Path to a test key store directory.
-   default value: null
+   default value: "/keys"
 
 InitConfig
 ^^^^^^^^^^
@@ -182,7 +230,7 @@ InitConfig
    default value: 30303
 
  EnableUnsecuredDevWallet
-   If 'true' then it enables thewallet / key store in the application.
+   If 'true' then it enables the wallet / key store in the application.
    default value: false
 
  GenesisHash
@@ -233,6 +281,10 @@ InitConfig
    If 'false' then the node does not download/process new blocks..
    default value: true
 
+ PubSubEnabled
+   If 'true' then it enables the Kafka producer which can be configured to stream the transactions data.
+   default value: false
+
  StaticNodesPath
    Path to the file with a list of static nodes.
    default value: "Data/static-nodes.json"
@@ -430,6 +482,30 @@ Sample configuration (mainnet)
           "CodeDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
           "CodeDbWriteBufferNumber" : [MISSING_DOCS],
           "CodeDbWriteBufferSize" : [MISSING_DOCS],
+          "ConfigsDbBlockCacheSize" : [MISSING_DOCS],
+          "ConfigsDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
+          "ConfigsDbWriteBufferNumber" : [MISSING_DOCS],
+          "ConfigsDbWriteBufferSize" : [MISSING_DOCS],
+          "ConsumerDepositApprovalsDbBlockCacheSize" : [MISSING_DOCS],
+          "ConsumerDepositApprovalsDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
+          "ConsumerDepositApprovalsDbWriteBufferNumber" : [MISSING_DOCS],
+          "ConsumerDepositApprovalsDbWriteBufferSize" : [MISSING_DOCS],
+          "ConsumerReceiptsDbBlockCacheSize" : [MISSING_DOCS],
+          "ConsumerReceiptsDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
+          "ConsumerReceiptsDbWriteBufferNumber" : [MISSING_DOCS],
+          "ConsumerReceiptsDbWriteBufferSize" : [MISSING_DOCS],
+          "ConsumerSessionsDbBlockCacheSize" : [MISSING_DOCS],
+          "ConsumerSessionsDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
+          "ConsumerSessionsDbWriteBufferNumber" : [MISSING_DOCS],
+          "ConsumerSessionsDbWriteBufferSize" : [MISSING_DOCS],
+          "DepositsDbBlockCacheSize" : [MISSING_DOCS],
+          "DepositsDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
+          "DepositsDbWriteBufferNumber" : [MISSING_DOCS],
+          "DepositsDbWriteBufferSize" : [MISSING_DOCS],
+          "EthRequestsDbBlockCacheSize" : [MISSING_DOCS],
+          "EthRequestsDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
+          "EthRequestsDbWriteBufferNumber" : [MISSING_DOCS],
+          "EthRequestsDbWriteBufferSize" : [MISSING_DOCS],
           "HeadersDbBlockCacheSize" : [MISSING_DOCS],
           "HeadersDbCacheIndexAndFilterBlocks" : [MISSING_DOCS],
           "HeadersDbWriteBufferNumber" : [MISSING_DOCS],
@@ -490,9 +566,9 @@ Sample configuration (mainnet)
       {
         "ConfigModule": "HiveConfig"
         "ConfigItems": {
-          "BlocksDir" : null,
-          "ChainFile" : null,
-          "KeysDir" : null,
+          "BlocksDir" : "/blocks",
+          "ChainFile" : "/chain.rlp",
+          "KeysDir" : "/keys",
         }
       },
       {
@@ -516,6 +592,7 @@ Sample configuration (mainnet)
           "P2PPort" : 30303,
           "PeerManagerEnabled" : true,
           "ProcessingEnabled" : true,
+          "PubSubEnabled" : false,
           "StaticNodesPath" : "Data/static-nodes.json",
           "StoreReceipts" : true,
           "StoreTraces" : false,

docs/source/jsonrpc.rst

@@ -40,6 +40,11 @@ clique
 
  - clique_propose(signer, vote)
 
+data
+^^^^
+
+ - data_streamBlocks(startBlockNumber, endBlockNumber)
+
 debug
 ^^^^^
 
@@ -67,7 +72,7 @@ debug
 
  - [NOT IMPLEMENTED]debug_traceBlockFromFile(fileName)
 
- - debug_traceTransaction(transactionHash)
+ - debug_traceTransaction(transactionHash, traceOptions)
 
  - debug_traceTransactionByBlockAndIndex(blockParameter, txIndex)
 
@@ -191,16 +196,24 @@ personal
 trace
 ^^^^^
 
+ - trace_block(numberOrTag)
+
  - [NOT IMPLEMENTED]trace_call(message, traceTypes, numberOrTag)
 
  - [NOT IMPLEMENTED]trace_callMany(calls)
 
+ - [NOT IMPLEMENTED]trace_filter(fromBlock, toBlock, toAddress, after, count)
+
+ - [NOT IMPLEMENTED]trace_get(txHash, positions)
+
  - [NOT IMPLEMENTED]trace_rawTransaction(data, traceTypes)
 
  - trace_replayBlockTransactions(numberOrTag, traceTypes)
 
  - trace_replayTransaction(txHash, traceTypes)
 
+ - trace_transaction(txHash)
+
 txpool
 ^^^^^^
 

src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs

@@ -240,8 +240,7 @@ protected async Task RunTest(BlockchainTest test, Stopwatch stopwatch = null)
             IRewardCalculator rewardCalculator = new RewardCalculator(specProvider);
 
             IEthereumEcdsa ecdsa = new EthereumEcdsa(specProvider, _logManager);
-            ITxPool transactionPool = new TxPool(NullTxStorage.Instance,
-                new PendingTxThresholdValidator(), new Timestamp(), ecdsa, specProvider, _logManager);
+            ITxPool transactionPool = new TxPool(NullTxStorage.Instance, new Timestamp(), ecdsa, specProvider, new TxPoolConfig(), _logManager);
             IReceiptStorage receiptStorage = NullReceiptStorage.Instance;
             IBlockTree blockTree = new BlockTree(new MemDb(), new MemDb(), new MemDb(), specProvider, transactionPool, _logManager);
             IBlockhashProvider blockhashProvider = new BlockhashProvider(blockTree, _logManager);

src/Nethermind/Nethermind.Blockchain.Test/IntegrationTests.cs

@@ -62,11 +62,10 @@ public async Task Can_process_mined_blocks()
 
             /* store & validation */
 
-            EthereumEcdsa ethereumEcdsa = new EthereumEcdsa(specProvider, logManager);
+            EthereumEcdsa ecdsa = new EthereumEcdsa(specProvider, logManager);
             MemDb receiptsDb = new MemDb();
             MemDb traceDb = new MemDb();
-            TxPool txPool = new TxPool(new NullTxStorage(),
-                new PendingTxThresholdValidator(), new Timestamp(), ethereumEcdsa, specProvider, logManager);
+            TxPool txPool = new TxPool(NullTxStorage.Instance, Timestamp.Default, ecdsa, specProvider, new TxPoolConfig(), logManager);
             IReceiptStorage receiptStorage = new PersistentReceiptStorage(receiptsDb, specProvider, logManager);
             BlockTree blockTree = new BlockTree(new MemDb(), new MemDb(), new MemDb(), specProvider, txPool, logManager);
             Timestamp timestamp = new Timestamp();
@@ -82,7 +81,7 @@ public async Task Can_process_mined_blocks()
             StateProvider stateProvider = new StateProvider(stateDb, codeDb, logManager);
             StorageProvider storageProvider = new StorageProvider(stateDb, stateProvider, logManager);
 
-            TestTransactionsGenerator generator = new TestTransactionsGenerator(txPool, ethereumEcdsa, TimeSpan.FromMilliseconds(5 * timeMultiplier), NullLogManager.Instance);
+            TestTransactionsGenerator generator = new TestTransactionsGenerator(txPool, ecdsa, TimeSpan.FromMilliseconds(5 * timeMultiplier), NullLogManager.Instance);
             generator.Start();
 
             /* blockchain processing */
@@ -92,7 +91,7 @@ public async Task Can_process_mined_blocks()
             RewardCalculator rewardCalculator = new RewardCalculator(specProvider);
             BlockProcessor blockProcessor = new BlockProcessor(specProvider, blockValidator, rewardCalculator,
                 processor, stateDb, codeDb, traceDb, stateProvider, storageProvider, txPool, receiptStorage, logManager);
-            BlockchainProcessor blockchainProcessor = new BlockchainProcessor(blockTree, blockProcessor, new TxSignaturesRecoveryStep(ethereumEcdsa, NullTxPool.Instance, LimboLogs.Instance), logManager, false, false);
+            BlockchainProcessor blockchainProcessor = new BlockchainProcessor(blockTree, blockProcessor, new TxSignaturesRecoveryStep(ecdsa, NullTxPool.Instance, LimboLogs.Instance), logManager, false, false);
 
             /* load ChainSpec and init */
             ChainSpecLoader loader = new ChainSpecLoader(new EthereumJsonSerializer());

src/Nethermind/Nethermind.Blockchain.Test/PendingTransactionThresholdValidatorTests.cs

@@ -67,8 +67,13 @@ private PendingTxThresholdValidator GetValidator(int obsoletePendingTransactionI
             _obsoletePendingTransactionInterval = obsoletePendingTransactionInterval;
             _removePendingTransactionInterval = removePendingTransactionInterval;
 
-            return new PendingTxThresholdValidator(_obsoletePendingTransactionInterval,
-                _removePendingTransactionInterval);
+            TxPoolConfig config = new TxPoolConfig
+            {
+                ObsoletePendingTransactionInterval = _obsoletePendingTransactionInterval,
+                RemovePendingTransactionInterval = _removePendingTransactionInterval
+            };
+
+            return new PendingTxThresholdValidator(config);
         }
 
         private Transaction GetTransaction(DateTime utcNow, int createdSecondsAgo = 0)

src/Nethermind/Nethermind.Blockchain.Test/Synchronization/SyncThreadTests.cs

@@ -256,7 +256,7 @@ private SyncTestContext CreateSyncManager(int index)
             var receiptStorage = new InMemoryReceiptStorage();
 
             var ecdsa = new EthereumEcdsa(specProvider, logManager);
-            var txPool = new TxPool(new InMemoryTxStorage(), new PendingTxThresholdValidator(), new Timestamp(), ecdsa, specProvider, logManager);
+            var txPool = new TxPool(new InMemoryTxStorage(), new Timestamp(), ecdsa, specProvider, new TxPoolConfig(), logManager);
             var tree = new BlockTree(blockDb, headerDb, blockInfoDb, specProvider, txPool, logManager);
             var blockhashProvider = new BlockhashProvider(tree, LimboLogs.Instance);
             var virtualMachine = new VirtualMachine(stateProvider, storageProvider, blockhashProvider, logManager);

src/Nethermind/Nethermind.Blockchain.Test/TransactionPoolTests.cs

@@ -227,8 +227,8 @@ private IDictionary<ISyncPeer, PrivateKey> GetPeers(int limit = 100)
         }
 
         private TxPool CreatePool(ITxStorage txStorage)
-            => new TxPool(txStorage, new PendingTxThresholdValidator(),
-                new Timestamp(), _ethereumEcdsa, _specProvider, _logManager);
+            => new TxPool(txStorage,
+                Timestamp.Default, _ethereumEcdsa, _specProvider, new TxPoolConfig(), _logManager);
 
         private ISyncPeer GetPeer(PublicKey publicKey)
             => new SyncPeerMock(_remoteBlockTree, publicKey);

src/Nethermind/Nethermind.Blockchain/BlockTree.cs

@@ -594,6 +594,11 @@ public AddBlockResult SuggestHeader(BlockHeader header)
 
         public AddBlockResult SuggestBlock(Block block, bool shouldProcess = true)
         {
+            if (Genesis == null && !block.IsGenesis)
+            {
+                throw new InvalidOperationException("Block tree should be initialized with genesis before suggesting other blocks.");
+            }
+
             return Suggest(block, block.Header, shouldProcess);
         }
 

src/Nethermind/Nethermind.Blockchain/TxPools/PendingTxThresholdValidator.cs

@@ -16,6 +16,7 @@
  * along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
  */
 
+using System;
 using Nethermind.Dirichlet.Numerics;
 
 namespace Nethermind.Blockchain.TxPools
@@ -25,11 +26,12 @@ public class PendingTxThresholdValidator : IPendingTxThresholdValidator
         private readonly int _obsoletePendingTransactionInterval;
         private readonly int _removePendingTransactionInterval;
 
-        public PendingTxThresholdValidator(int obsoletePendingTransactionInterval = 15,
-            int removePendingTransactionInterval = 600)
+        public PendingTxThresholdValidator(ITxPoolConfig txPoolConfig)
         {
-            _obsoletePendingTransactionInterval = obsoletePendingTransactionInterval;
-            _removePendingTransactionInterval = removePendingTransactionInterval;
+            if(txPoolConfig == null) throw new ArgumentNullException(nameof(txPoolConfig));
+
+            _obsoletePendingTransactionInterval = txPoolConfig.ObsoletePendingTransactionInterval;
+            _removePendingTransactionInterval = txPoolConfig.RemovePendingTransactionInterval;
         }
 
         public bool IsObsolete(UInt256 currentTimestamp, UInt256 transactionTimestamp)

src/Nethermind/Nethermind.Blockchain/TxPools/TxPool.cs

@@ -44,8 +44,6 @@ public class TxPool : ITxPool
             new ConcurrentDictionary<Keccak, bool>();
         private readonly ConcurrentDictionary<Type, ITxFilter> _filters =
             new ConcurrentDictionary<Type, ITxFilter>();
-        private readonly ITxStorage _transactionStorage;
-        private readonly IPendingTxThresholdValidator _pendingTransactionThresholdValidator;
 
         private readonly ITxStorage _txStorage;
         private readonly IPendingTxThresholdValidator _pendingTxThresholdValidator;
@@ -59,18 +57,21 @@ public class TxPool : ITxPool
         private readonly Timer _ownTimer;
 
         public TxPool(ITxStorage txStorage,
-            IPendingTxThresholdValidator pendingTxThresholdValidator,
-            ITimestamp timestamp, IEthereumEcdsa ecdsa, ISpecProvider specProvider, ILogManager logManager,
-            int removePendingTransactionInterval = 600,
-            int peerNotificationThreshold = 20)
+            ITimestamp timestamp,
+            IEthereumEcdsa ecdsa,
+            ISpecProvider specProvider,
+            ITxPoolConfig txPoolConfig,
+            ILogManager logManager)
         {
+            int removePendingTransactionInterval = txPoolConfig.RemovePendingTransactionInterval;
+            _peerNotificationThreshold = txPoolConfig.PeerNotificationThreshold;
             _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager));
             _txStorage = txStorage ?? throw new ArgumentNullException(nameof(txStorage));
-            _pendingTxThresholdValidator = pendingTxThresholdValidator;
             _timestamp = timestamp ?? throw new ArgumentNullException(nameof(timestamp));
             _ecdsa = ecdsa ?? throw new ArgumentNullException(nameof(ecdsa));
             _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider));
-            _peerNotificationThreshold = peerNotificationThreshold;
+
+            _pendingTxThresholdValidator = new PendingTxThresholdValidator(txPoolConfig);
             if (removePendingTransactionInterval <= 0)
             {
                 return;

src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs

@@ -88,7 +88,7 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f
                 MemDb headersDb = new MemDb();
                 MemDb blockInfoDb = new MemDb();
 
-                TxPool txPool = new TxPool(new InMemoryTxStorage(), new PendingTxThresholdValidator(), _timestamp, _ethereumEcdsa, GoerliSpecProvider.Instance, _logManager);
+                TxPool txPool = new TxPool(new InMemoryTxStorage(), _timestamp, _ethereumEcdsa, GoerliSpecProvider.Instance, new TxPoolConfig(), _logManager);
                 _pools[privateKey] = txPool;
 
                 BlockTree blockTree = new BlockTree(blocksDb, headersDb, blockInfoDb, GoerliSpecProvider.Instance, txPool, nodeLogManager);