Trace/db traces

.github/workflows/run-nethermind-tests-with-code-coverage.yml

@@ -175,6 +175,9 @@ jobs:
     - name: Nethermind.Sockets.Test
       run: |
         dotnet test -c Release $EXCLUDE_TEST_PROJECTS src/Nethermind/Nethermind.Sockets.Test
+    - name: Nethermind.JsonRpc.TraceStore.Tests
+      run: |
+        dotnet test -c Release $EXCLUDE_TEST_PROJECTS src/Nethermind/Nethermind.JsonRpc.TraceStore.Tests        
     - name: Upload Codecov Report
       if: matrix.os == 'ubuntu-latest'
       uses: actions/upload-artifact@v2

.github/workflows/run-nethermind-tests.yml

@@ -45,6 +45,9 @@ jobs:
     - name: Nethermind.AccountAbstraction.Test
       run: |
         dotnet test src/Nethermind/Nethermind.AccountAbstraction.Test -c ${{ env.BUILD_CONFIG }}
+    - name: Nethermind.JsonRpc.TraceStore.Tests
+      run: |
+        dotnet test src/Nethermind/Nethermind.JsonRpc.TraceStore.Tests -c ${{ env.BUILD_CONFIG }}
 
   neth-tests2:
     name: Running Nethermind Tests 2

scripts/deployment/archive-packages.sh

@@ -35,11 +35,11 @@ dotnet build -c release Nethermind.Runner.csproj
 
 cd $RELEASE_DIRECTORY
 
-cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev}.dll $LIN_RELEASE/plugins
-cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev}.dll $OSX_RELEASE/plugins
-cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev}.dll $WIN_RELEASE/plugins
-cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev}.dll $LIN_ARM64_RELEASE/plugins
-cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev}.dll $OSX_ARM64_RELEASE/plugins
+cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev,JsonRpc.TraceStore}.dll $LIN_RELEASE/plugins
+cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev,JsonRpc.TraceStore}.dll $OSX_RELEASE/plugins
+cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev,JsonRpc.TraceStore}.dll $WIN_RELEASE/plugins
+cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev,JsonRpc.TraceStore}.dll $LIN_ARM64_RELEASE/plugins
+cp $RELEASE_DIRECTORY/$RELEASE_PATH/Nethermind.{Api,HealthChecks,EthStats,Merge.Plugin,Mev,JsonRpc.TraceStore}.dll $OSX_ARM64_RELEASE/plugins
 
 cd $LIN_RELEASE && zip -r $LIN-$VERSION-$COMMIT_HASH-$DATE.zip . && cd ..
 cd $OSX_RELEASE && zip -r $OSX-$VERSION-$COMMIT_HASH-$DATE.zip . && cd ..

src/Nethermind/Directory.Build.props

@@ -11,7 +11,7 @@
     <Copyright>Demerzel Solutions Limited</Copyright>
     <Product>Nethermind</Product>
     <SourceRevisionId Condition="'$(Commit)' != ''">$(Commit.Substring(0, 8))</SourceRevisionId>
-    <VersionPrefix>1.14.6</VersionPrefix>
+    <VersionPrefix>1.14.7</VersionPrefix>
     <VersionSuffix></VersionSuffix>
   </PropertyGroup>
 

src/Nethermind/Nethermind.Consensus/BlocksConfig.cs

@@ -1,19 +1,5 @@
-//  Copyright (c) 2021 Demerzel Solutions Limited
-//  This file is part of the Nethermind library.
-// 
-//  The Nethermind library is free software: you can redistribute it and/or modify
-//  it under the terms of the GNU Lesser General Public License as published by
-//  the Free Software Foundation, either version 3 of the License, or
-//  (at your option) any later version.
-// 
-//  The Nethermind library is distributed in the hope that it will be useful,
-//  but WITHOUT ANY WARRANTY; without even the implied warranty of
-//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//  GNU Lesser General Public License for more details.
-// 
-//  You should have received a copy of the GNU Lesser General Public License
-//  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
-// 
+// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
 using System.Reflection;

src/Nethermind/Nethermind.Consensus/IBlocksConfig.cs

@@ -1,18 +1,5 @@
-//  Copyright (c) 2021 Demerzel Solutions Limited
-//  This file is part of the Nethermind library.
-// 
-//  The Nethermind library is free software: you can redistribute it and/or modify
-//  it under the terms of the GNU Lesser General Public License as published by
-//  the Free Software Foundation, either version 3 of the License, or
-//  (at your option) any later version.
-// 
-//  The Nethermind library is distributed in the hope that it will be useful,
-//  but WITHOUT ANY WARRANTY; without even the implied warranty of
-//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//  GNU Lesser General Public License for more details.
-// 
-//  You should have received a copy of the GNU Lesser General Public License
-//  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
+// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
 
 using System.Collections.Generic;
 using System.Reflection;

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -223,10 +223,10 @@ protected virtual TxReceipt[] ProcessBlock(
             _receiptsTracer.SetOtherTracer(blockTracer);
             _receiptsTracer.StartNewBlockTrace(block);
             TxReceipt[] receipts = _blockTransactionsExecutor.ProcessTransactions(block, options, _receiptsTracer, spec);
-            _receiptsTracer.EndBlockTrace();
 
             block.Header.ReceiptsRoot = receipts.GetReceiptsRoot(spec, block.ReceiptsRoot);
             ApplyMinerRewards(block, blockTracer, spec);
+            _receiptsTracer.EndBlockTrace();
 
             _stateProvider.Commit(spec);
             _stateProvider.RecalculateStateRoot();

src/Nethermind/Nethermind.Consensus/Processing/BlockRef.cs

@@ -46,5 +46,7 @@ public bool Resolve(IBlockTree blockTree)
 
             return true;
         }
+
+        public override string ToString() => Block?.ToString() ?? BlockHash.ToString();
     }
 }

src/Nethermind/Nethermind.Core.Test/Collections/ArrayPoolListTests.cs

@@ -175,5 +175,18 @@ public void Set_should_throw(int item)
             Action action = () => list[item] = 1;
             action.Should().Throw<ArgumentOutOfRangeException>();
         }
+
+        [TestCase(1, 16)]
+        [TestCase(14, 16)]
+        [TestCase(15, 32)]
+        [TestCase(20, 32)]
+        [TestCase(100, 128)]
+        public void AddRange_should_expand(int items, int expectedCapacity)
+        {
+            ArrayPoolList<int> list = new(16) { 0, 1 };
+            list.AddRange(Enumerable.Range(2, items));
+            list.Should().BeEquivalentTo(Enumerable.Range(0, items + 2));
+            list.Capacity.Should().Be(expectedCapacity);
+        }
     }
 }

src/Nethermind/Nethermind.Core/BlockInfo.cs

@@ -76,5 +76,7 @@ public bool IsBeaconInfo
         /// This property is not serialized
         /// </summary>
         public long BlockNumber { get; set; }
+
+        public override string ToString() => BlockHash.ToString();
     }
 }

src/Nethermind/Nethermind.Core/Collections/ArrayPoolList.cs

@@ -7,15 +7,13 @@
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Threading;
-using System.Threading.Tasks;
-using Nethermind.Core.Crypto;
 
 namespace Nethermind.Core.Collections
 {
     public class ArrayPoolList<T> : IList<T>, IReadOnlyList<T>, IDisposable
     {
         private readonly ArrayPool<T> _arrayPool;
-        private T[] _array;
+        protected T[] _array;
         private int _count = 0;
         private int _capacity;
         private bool _disposed;
@@ -60,6 +58,13 @@ public void Add(T item)
             _array[_count++] = item;
         }
 
+        public void AddRange(Span<T> items)
+        {
+            GuardResize(items.Length);
+            items.CopyTo(_array.AsSpan(_count, items.Length));
+            _count += items.Length;
+        }
+
         public void Clear()
         {
             _count = 0;
@@ -100,12 +105,17 @@ public void Insert(int index, T item)
             _count++;
         }
 
-        private void GuardResize()
+        private void GuardResize(int itemsToAdd = 1)
         {
             GuardDispose();
-            if (_count == _capacity)
+            int newCount = _count + itemsToAdd;
+            if (newCount > _capacity)
             {
                 int newCapacity = _capacity * 2;
+                while (newCount > newCapacity)
+                {
+                    newCapacity *= 2;
+                }
                 T[] newArray = _arrayPool.Rent(newCapacity);
                 _array.CopyTo(newArray, 0);
                 T[] oldArray = Interlocked.Exchange(ref _array, newArray);
@@ -198,5 +208,7 @@ public void Dispose()
                 _disposed = true;
             }
         }
+
+        public Span<T> AsSpan() => _array.AsSpan(0, _count);
     }
 }

src/Nethermind/Nethermind.Core/Collections/ByteArrayPoolList.cs

@@ -0,0 +1,25 @@
+// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Buffers;
+using System.Collections.Generic;
+using System.IO;
+
+namespace Nethermind.Core.Collections;
+
+public class ByteArrayPoolList : ArrayPoolList<byte>
+{
+    public ByteArrayPoolList(int capacity) : base(capacity)
+    {
+    }
+
+    public ByteArrayPoolList(int capacity, IEnumerable<byte> enumerable) : base(capacity, enumerable)
+    {
+    }
+
+    public ByteArrayPoolList(ArrayPool<byte> arrayPool, int capacity) : base(arrayPool, capacity)
+    {
+    }
+
+    public MemoryStream AsMemoryStream() => new(_array, 0, Count);
+}

src/Nethermind/Nethermind.Core/Extensions/Bytes.cs

@@ -504,20 +504,8 @@ public static void StreamHex(this byte[] bytes, StreamWriter streamWriter)
             }
         }
 
-        public static string ToHexString(this byte[] bytes, bool withZeroX)
-        {
-            return ToHexString(bytes, withZeroX, false, false);
-        }
-
-        public static string ToHexString(this byte[] bytes, bool withZeroX, bool noLeadingZeros)
-        {
-            return ToHexString(bytes, withZeroX, noLeadingZeros, false);
-        }
-
-        public static string ToHexString(this byte[] bytes, bool withZeroX, bool noLeadingZeros, bool withEip55Checksum)
-        {
-            return ByteArrayToHexViaLookup32(bytes, withZeroX, noLeadingZeros, withEip55Checksum);
-        }
+        public static string ToHexString(this byte[] bytes, bool withZeroX, bool noLeadingZeros = false, bool withEip55Checksum = false) =>
+            ByteArrayToHexViaLookup32(bytes, withZeroX, noLeadingZeros, withEip55Checksum);
 
         private struct StateSmall
         {

src/Nethermind/Nethermind.Core/Resettables/ResettableList.cs

@@ -7,9 +7,9 @@
 
 namespace Nethermind.Core.Resettables;
 
-public class ResettableList<T> : IList<T>
+public class ResettableList<T> : IList<T>, IReadOnlyCollection<T>
 {
-    private List<T> _wrapped;
+    private readonly List<T> _wrapped;
     private readonly int _startCapacity;
     private readonly int _resetRatio;
     private int _currentCapacity;

src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs

@@ -196,9 +196,11 @@ protected internal void UpdateWriteMetrics()
             propertyName);
         try
         {
-            return (T?)dbConfig.GetType()
-                .GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance)?
-                .GetValue(dbConfig);
+            Type type = dbConfig.GetType();
+            PropertyInfo? propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
+            // if no custom db property default to generic one
+            propertyInfo ??= type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
+            return (T?)propertyInfo?.GetValue(dbConfig);
         }
         catch (Exception e)
         {

src/Nethermind/Nethermind.Db/RocksDbInitializer.cs

@@ -68,9 +68,6 @@ protected async Task InitAllAsync()
             await Task.WhenAll(allInitializers);
         }
 
-        protected static string GetTitleDbName(string dbName)
-        {
-            return char.ToUpper(dbName[0]) + dbName.Substring(1);
-        }
+        protected static string GetTitleDbName(string dbName) => char.ToUpper(dbName[0]) + dbName.Substring(1);
     }
 }

src/Nethermind/Nethermind.Evm/Tracing/BlockTracerBase.cs

@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using Nethermind.Core;
 using Nethermind.Core.Crypto;
+using Nethermind.Core.Resettables;
 using Nethermind.Int256;
 
 namespace Nethermind.Evm.Tracing
@@ -16,13 +17,13 @@ public abstract class BlockTracerBase<TTrace, TTracer> : IBlockTracer where TTra
 
         protected BlockTracerBase()
         {
-            TxTraces = new List<TTrace>();
+            TxTraces = new ResettableList<TTrace>();
         }
 
         protected BlockTracerBase(Keccak? txHash)
         {
             _txHash = txHash;
-            TxTraces = new List<TTrace>();
+            TxTraces = new ResettableList<TTrace>();
         }
 
         private TTracer? CurrentTxTracer { get; set; }
@@ -36,7 +37,10 @@ public virtual void ReportReward(Address author, string rewardType, UInt256 rewa
         {
         }
 
-        public abstract void StartNewBlockTrace(Block block);
+        public virtual void StartNewBlockTrace(Block block)
+        {
+            TxTraces.Reset();
+        }
 
         ITxTracer IBlockTracer.StartNewTxTrace(Transaction? tx)
         {
@@ -58,18 +62,15 @@ void IBlockTracer.EndTxTrace()
             }
         }
 
-        public abstract void EndBlockTrace();
+        public virtual void EndBlockTrace() { }
 
         protected virtual bool ShouldTraceTx(Transaction? tx)
         {
             return IsTracingEntireBlock || tx?.Hash == _txHash;
         }
 
-        protected List<TTrace> TxTraces { get; }
+        protected ResettableList<TTrace> TxTraces { get; }
 
-        public IReadOnlyCollection<TTrace> BuildResult()
-        {
-            return TxTraces;
-        }
+        public IReadOnlyCollection<TTrace> BuildResult() => TxTraces;
     }
 }

src/Nethermind/Nethermind.Evm/Tracing/GethStyle/GethLikeBlockTracer.cs

@@ -24,13 +24,5 @@ public GethLikeBlockTracer(Keccak txHash, GethTraceOptions options)
         protected override GethLikeTxTracer OnStart(Transaction? tx) => new(_options);
 
         protected override GethLikeTxTrace OnEnd(GethLikeTxTracer txTracer) => txTracer.BuildResult();
-
-        public override void StartNewBlockTrace(Block block)
-        {
-        }
-
-        public override void EndBlockTrace()
-        {
-        }
     }
 }

src/Nethermind/Nethermind.Evm/Tracing/IBlockTracer.cs

@@ -15,16 +15,45 @@ namespace Nethermind.Evm.Tracing
     /// </remarks>
     public interface IBlockTracer
     {
+        /// <summary>
+        /// Is reward state change traced
+        /// </summary>
+        /// <remarks>
+        /// Controls
+        /// - <see cref="ReportReward"/>
+        /// </remarks>
         bool IsTracingRewards { get; }
 
+        /// <summary>
+        /// Reports rewards for bock.
+        /// </summary>
+        /// <param name="author">Author/coinbase for reward.</param>
+        /// <param name="rewardType">Type of reward.</param>
+        /// <param name="rewardValue">Value of reward.</param>
+        /// <remarks>Depends on <see cref="IsTracingRewards"/></remarks>
         void ReportReward(Address author, string rewardType, UInt256 rewardValue);
 
+        /// <summary>
+        /// Starts a trace for new block.
+        /// </summary>
+        /// <param name="block">Block to be traced.</param>
         void StartNewBlockTrace(Block block);
 
+        /// <summary>
+        /// Starts new transaction trace in a block.
+        /// </summary>
+        /// <param name="tx">Transaction this trace is started for. Null if it's reward trace.</param>
+        /// <returns>Returns tracer for transaction.</returns>
         ITxTracer StartNewTxTrace(Transaction? tx);
 
+        /// <summary>
+        /// Ends last transaction trace <see cref="StartNewTxTrace"/>.
+        /// </summary>
         void EndTxTrace();
 
+        /// <summary>
+        /// Ends block trace <see cref="StartNewBlockTrace"/>.
+        /// </summary>
         void EndBlockTrace();
     }
 }