Split libsilkworm into silkworm_core & silkworm_db

CMakeLists.txt

@@ -80,9 +80,9 @@ if(MSVC)
     add_compile_definitions(USE_NUM_NONE USE_FIELD_INV_BUILTIN USE_SCALAR_INV_BUILTIN)
   endif()
   add_compile_definitions(USE_FIELD_10X26 USE_SCALAR_8X32)
- else()
-  add_compile_definitions(USE_FIELD_INV_BUILTIN USE_SCALAR_INV_BUILTIN)
-  add_compile_definitions(USE_NUM_NONE USE_FIELD_5X52 USE_SCALAR_4X64 HAVE___INT128)
+else()
+  add_compile_definitions(USE_NUM_NONE USE_FIELD_INV_BUILTIN USE_SCALAR_INV_BUILTIN)
+  add_compile_definitions(USE_FIELD_5X52 USE_SCALAR_4X64 HAVE___INT128)
 endif()
 add_compile_definitions(ECMULT_WINDOW_SIZE=15 ECMULT_GEN_PREC_BITS=4 USE_ENDOMORPHISM)
 add_compile_definitions(ENABLE_MODULE_RECOVERY)
@@ -153,7 +153,8 @@ target_include_directories(evmone PUBLIC evmone/lib/evmone)
 target_link_libraries(evmone PUBLIC evmc intx::intx PRIVATE ethash::keccak)
 
 # Silkworm itself
-add_subdirectory(silkworm)
+add_subdirectory(core)
+add_subdirectory(db)
 add_subdirectory(tg_api)
 if(NOT SILKWORM_HAS_PARENT)
   add_subdirectory(cmd)

cmd/CMakeLists.txt

@@ -15,7 +15,7 @@
 ]]
 
 find_package(absl CONFIG REQUIRED)
-#find_package(Boost CONFIG REQUIRED COMPONENTS filesystem)
+find_package(Boost CONFIG REQUIRED COMPONENTS filesystem)
 
 if(MSVC)
   add_link_options(/STACK:10000000)
@@ -24,37 +24,37 @@ elseif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 endif()
 
 add_executable(check_changes check_changes.cpp)
-target_link_libraries(check_changes PRIVATE silkworm absl::flags_parse absl::time)
+target_link_libraries(check_changes PRIVATE silkworm_db absl::flags_parse absl::time Boost::filesystem)
 
 add_executable(scan_txs scan_txs.cpp)
-target_link_libraries(scan_txs PRIVATE silkworm absl::flags_parse absl::time)
+target_link_libraries(scan_txs PRIVATE silkworm_db absl::flags_parse absl::time Boost::filesystem)
 
 find_package(CLI11 CONFIG REQUIRED)
 
 add_executable(check_senders check_senders.cpp)
-target_link_libraries(check_senders PRIVATE silkworm CLI11::CLI11)
+target_link_libraries(check_senders PRIVATE silkworm_db CLI11::CLI11 Boost::filesystem)
 
 add_executable(dbtool dbtool.cpp)
-target_link_libraries(dbtool PRIVATE silkworm CLI11::CLI11)
+target_link_libraries(dbtool PRIVATE silkworm_db CLI11::CLI11 Boost::filesystem)
 
 add_executable(execute execute.cpp)
-target_link_libraries(execute PRIVATE silkworm silkworm_tg_api CLI11::CLI11)
+target_link_libraries(execute PRIVATE silkworm_db silkworm_tg_api CLI11::CLI11 Boost::filesystem)
 target_include_directories(execute PRIVATE ${CMAKE_SOURCE_DIR})
 
 # Ethereum Consensus Tests
 find_package(nlohmann_json CONFIG REQUIRED)
 add_compile_definitions(SILKWORM_CONSENSUS_TEST_DIR="${CMAKE_SOURCE_DIR}/tests")
 add_executable(consensus consensus.cpp)
-target_link_libraries(consensus PRIVATE silkworm nlohmann_json::nlohmann_json)
+target_link_libraries(consensus PRIVATE silkworm_core nlohmann_json::nlohmann_json Boost::filesystem)
 
 # Unit tests
 enable_testing()
 
 find_package(Catch2 CONFIG REQUIRED)
 
-file(GLOB_RECURSE SILKWORM_TESTS CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/silkworm/*_test.cpp")
+file(GLOB_RECURSE SILKWORM_TESTS CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/*/silkworm/*_test.cpp")
 add_executable(unit_test unit_test.cpp ${SILKWORM_TESTS})
-target_link_libraries(unit_test silkworm Catch2::Catch2)
+target_link_libraries(unit_test silkworm_db Catch2::Catch2)
 
 include(CTest)
 include(Catch)
@@ -63,4 +63,4 @@ catch_discover_tests(unit_test)
 # Benchmarks
 find_package(benchmark CONFIG REQUIRED)
 add_executable(benchmark_precompile benchmark_precompile.cpp)
-target_link_libraries(benchmark_precompile silkworm benchmark::benchmark)
+target_link_libraries(benchmark_precompile silkworm_core benchmark::benchmark)

cmd/check_changes.cpp

@@ -20,8 +20,10 @@
 #include <absl/flags/usage.h>
 #include <absl/time/time.h>
 
+#include <boost/filesystem.hpp>
 #include <iostream>
 #include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/buffer.hpp>
 #include <silkworm/execution/execution.hpp>
 
 using namespace evmc::literals;

cmd/consensus.cpp

@@ -14,6 +14,7 @@
    limitations under the License.
 */
 
+#include <boost/filesystem.hpp>
 #include <exception>
 #include <fstream>
 #include <iostream>
@@ -25,6 +26,7 @@
 #include <silkworm/execution/processor.hpp>
 #include <silkworm/rlp/decode.hpp>
 #include <silkworm/state/intra_block_state.hpp>
+#include <silkworm/state/memory_buffer.hpp>
 #include <silkworm/types/block.hpp>
 #include <string>
 #include <string_view>
@@ -276,6 +278,8 @@ Status run_block(const nlohmann::json& b, const ChainConfig& config, IntraBlockS
         return kFailed;
     }
 
+    processor.evm().state().write_to_db(block_number);
+
     if (invalid) {
         std::cout << "Invalid block executed successfully\n";
         std::cout << "Expected: " << b["expectException"] << "\n";
@@ -344,12 +348,12 @@ Status blockchain_test(const nlohmann::json& j, std::optional<ChainConfig>) {
     Block genesis_block;
     rlp::decode(genesis_view, genesis_block);
 
-    db::Buffer db{nullptr};
-    db.insert_header(genesis_block.header);
+    MemoryBuffer buffer;
+    buffer.insert_header(genesis_block.header);
 
     std::string network{j["network"].get<std::string>()};
     const ChainConfig& config{kNetworkConfig.at(network)};
-    IntraBlockState state{db};
+    IntraBlockState state{buffer};
     init_pre_state(j["pre"], state);
 
     for (const auto& b : j["blocks"]) {
@@ -541,15 +545,15 @@ int main() {
     for (auto i = fs::recursive_directory_iterator(kBlockchainDir); i != fs::recursive_directory_iterator{}; ++i) {
         if (kSlowTests.count(*i) || kFailingTests.count(*i)) {
             i.disable_recursion_pending();
-        } else if (boost::filesystem::is_regular_file(i->path())) {
+        } else if (fs::is_regular_file(i->path())) {
             res += run_test_file(*i, blockchain_test);
         }
     }
 
     for (auto i = fs::recursive_directory_iterator(kTransactionDir); i != fs::recursive_directory_iterator{}; ++i) {
         if (kFailingTests.count(*i)) {
             i.disable_recursion_pending();
-        } else if (boost::filesystem::is_regular_file(i->path())) {
+        } else if (fs::is_regular_file(i->path())) {
             res += run_test_file(*i, transaction_test);
         }
     }
@@ -564,5 +568,5 @@ int main() {
     }
     std::cout << ", " << res.skipped << " skipped\n";
 
-    return res.failed;
+    return static_cast<int>(res.failed);
 }

cmd/dbtool.cpp

@@ -17,6 +17,7 @@
 #include <CLI/CLI.hpp>
 #include <boost/bind.hpp>
 #include <boost/endian/conversion.hpp>
+#include <boost/filesystem.hpp>
 #include <boost/format.hpp>
 #include <csignal>
 #include <iostream>

cmd/execute.cpp

@@ -16,6 +16,7 @@
 
 #include <CLI/CLI.hpp>
 #include <boost/endian/conversion.hpp>
+#include <boost/filesystem.hpp>
 #include <limits>
 #include <silkworm/common/log.hpp>
 #include <silkworm/db/access_layer.hpp>

cmd/scan_txs.cpp

@@ -23,6 +23,7 @@
 #include <boost/filesystem.hpp>
 #include <iostream>
 #include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/buffer.hpp>
 #include <silkworm/execution/execution.hpp>
 
 ABSL_FLAG(std::string, datadir, silkworm::db::default_path(), "chain DB path");

silkworm/CMakeLists.txt → core/CMakeLists.txt

@@ -15,24 +15,19 @@
 ]]
 
 find_package(absl CONFIG REQUIRED)
-find_package(Boost CONFIG REQUIRED COMPONENTS filesystem)
+find_package(Boost CONFIG REQUIRED)
 find_package(cryptopp CONFIG REQUIRED)
 find_package(Microsoft.GSL CONFIG REQUIRED)
 
-file(GLOB_RECURSE SILKWORM_SRC CONFIGURE_DEPENDS "*.cpp" "*.hpp" "*.c" "*.h")
-list(FILTER SILKWORM_SRC EXCLUDE REGEX "_test\.cpp$")
+file(GLOB_RECURSE SILKWORM_CORE_SRC CONFIGURE_DEPENDS "*.cpp" "*.hpp" "*.c" "*.h")
+list(FILTER SILKWORM_CORE_SRC EXCLUDE REGEX "_test\.cpp$")
 
-add_library(silkworm ${SILKWORM_SRC})
+add_library(silkworm_core ${SILKWORM_CORE_SRC})
 get_filename_component(SILKWORM_MAIN_DIR ../ ABSOLUTE)
-target_include_directories(silkworm PUBLIC ${SILKWORM_MAIN_DIR})
+target_include_directories(silkworm_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${SILKWORM_MAIN_DIR})
 
-set(SILKWORM_PUBLIC_LIBS absl::flat_hash_map absl::flat_hash_set absl::btree Boost::filesystem
-                         lmdb evmc intx::intx ff Microsoft.GSL::GSL)
+set(SILKWORM_CORE_PUBLIC_LIBS absl::flat_hash_map absl::flat_hash_set
+                              evmc intx::intx ff Microsoft.GSL::GSL)
+set(SILKWORM_CORE_PRIVATE_LIBS evmone secp256k1 cryptopp-static ethash::keccak Boost::boost)
 
-set(SILKWORM_PRIVATE_LIBS evmone secp256k1 cryptopp-static ethash::keccak cborcpp)
-
-if(MSVC)
-  list(APPEND SILKWORM_PRIVATE_LIBS ntdll.lib)
-endif(MSVC)
-
-target_link_libraries(silkworm PUBLIC ${SILKWORM_PUBLIC_LIBS} PRIVATE ${SILKWORM_PRIVATE_LIBS})
+target_link_libraries(silkworm_core PUBLIC ${SILKWORM_CORE_PUBLIC_LIBS} PRIVATE ${SILKWORM_CORE_PRIVATE_LIBS})

silkworm/execution/execution.cpp → core/silkworm/execution/execution.cpp

@@ -16,14 +16,13 @@
 
 #include "execution.hpp"
 
-#include <silkworm/db/access_layer.hpp>
 #include <silkworm/trie/vector_root.hpp>
 
 #include "processor.hpp"
 
 namespace silkworm {
 
-std::vector<Receipt> execute_block(const Block& block, db::Buffer& buffer, const ChainConfig& config,
+std::vector<Receipt> execute_block(const Block& block, StateBuffer& buffer, const ChainConfig& config,
                                    AnalysisCache* analysis_cache) {
     const BlockHeader& header{block.header};
     uint64_t block_num{header.number};
@@ -50,6 +49,8 @@ std::vector<Receipt> execute_block(const Block& block, db::Buffer& buffer, const
         }
     }
 
+    processor.evm().state().write_to_db(block_num);
+
     return receipts;
 }
 

silkworm/execution/execution.hpp → core/silkworm/execution/execution.hpp

@@ -18,8 +18,9 @@
 #define SILKWORM_EXECUTION_EXECUTION_H_
 
 #include <silkworm/chain/config.hpp>
-#include <silkworm/db/buffer.hpp>
 #include <silkworm/execution/analysis_cache.hpp>
+#include <silkworm/state/buffer.hpp>
+#include <silkworm/types/block.hpp>
 #include <silkworm/types/receipt.hpp>
 #include <stdexcept>
 
@@ -37,7 +38,7 @@ class ValidationError : public std::runtime_error {
  *
  * For better performance use AnalysisCache.
  */
-std::vector<Receipt> execute_block(const Block& block, db::Buffer& buffer, const ChainConfig& config = kMainnetConfig,
+std::vector<Receipt> execute_block(const Block& block, StateBuffer& buffer, const ChainConfig& config = kMainnetConfig,
                                    AnalysisCache* analysis_cache = nullptr);
 
 }  // namespace silkworm

silkworm/execution/execution_test.cpp → core/silkworm/execution/execution_test.cpp

@@ -22,6 +22,7 @@
 #include <silkworm/chain/config.hpp>
 #include <silkworm/common/temp_dir.hpp>
 #include <silkworm/db/access_layer.hpp>
+#include <silkworm/db/buffer.hpp>
 #include <silkworm/db/chaindb.hpp>
 #include <silkworm/db/tables.hpp>
 #include <silkworm/execution/address.hpp>

silkworm/execution/processor.cpp → core/silkworm/execution/processor.cpp

@@ -156,8 +156,6 @@ std::vector<Receipt> ExecutionProcessor::execute_block() {
         evm_.state().destruct(kRipemdAddress);
     }
 
-    evm_.state().write_to_db(block_num);
-
     return receipts;
 }
 

silkworm/execution/processor.hpp → core/silkworm/execution/processor.hpp

@@ -37,6 +37,7 @@ class ExecutionProcessor {
     // precondition: txn.from must be recovered
     Receipt execute_transaction(const Transaction& txn);
 
+    /// Execute the block, but do not write to the DB yet
     std::vector<Receipt> execute_block();
 
     uint64_t cumulative_gas_used() const { return cumulative_gas_used_; }

silkworm/execution/processor_test.cpp → core/silkworm/execution/processor_test.cpp

@@ -19,6 +19,7 @@
 #include <catch2/catch.hpp>
 #include <evmc/evmc.hpp>
 #include <silkworm/common/temp_dir.hpp>
+#include <silkworm/db/buffer.hpp>
 #include <silkworm/db/chaindb.hpp>
 #include <silkworm/db/tables.hpp>
 

silkworm/db/state_buffer.hpp → core/silkworm/state/buffer.hpp

@@ -14,17 +14,23 @@
    limitations under the License.
 */
 
-#ifndef SILKWORM_DB_STATE_BUFFER_H_
-#define SILKWORM_DB_STATE_BUFFER_H_
+#ifndef SILKWORM_STATE_BUFFER_H_
+#define SILKWORM_STATE_BUFFER_H_
 
 #include <evmc/evmc.hpp>
 #include <optional>
 #include <silkworm/types/account.hpp>
+#include <silkworm/types/block.hpp>
 
-namespace silkworm::db {
+namespace silkworm {
 
 class StateBuffer {
   public:
+    StateBuffer() = default;
+
+    StateBuffer(const StateBuffer&) = delete;
+    StateBuffer& operator=(const StateBuffer&) = delete;
+
     virtual ~StateBuffer() = default;
 
     /** @name Readers */
@@ -67,6 +73,6 @@ class StateBuffer {
     ///@}
 };
 
-}  // namespace silkworm::db
+}  // namespace silkworm
 
-#endif  // SILKWORM_DB_STATE_BUFFER_H_
+#endif  // SILKWORM_STATE_BUFFER_H_

silkworm/state/intra_block_state.hpp → core/silkworm/state/intra_block_state.hpp

@@ -23,7 +23,7 @@
 #include <evmc/evmc.hpp>
 #include <intx/intx.hpp>
 #include <memory>
-#include <silkworm/db/state_buffer.hpp>
+#include <silkworm/state/buffer.hpp>
 #include <silkworm/state/delta.hpp>
 #include <silkworm/state/object.hpp>
 #include <silkworm/types/log.hpp>
@@ -32,13 +32,13 @@
 namespace silkworm {
 
 class IntraBlockState {
-   public:
+  public:
     class Snapshot {
-       public:
+      public:
         Snapshot(Snapshot&&) = default;
         Snapshot& operator=(Snapshot&&) = default;
 
-       private:
+      private:
         friend class IntraBlockState;
 
         Snapshot() = default;
@@ -51,9 +51,9 @@ class IntraBlockState {
     IntraBlockState(const IntraBlockState&) = delete;
     IntraBlockState& operator=(const IntraBlockState&) = delete;
 
-    explicit IntraBlockState(db::StateBuffer& db) noexcept : db_{db} {}
+    explicit IntraBlockState(StateBuffer& db) noexcept : db_{db} {}
 
-    db::StateBuffer& db() { return db_; }
+    StateBuffer& db() { return db_; }
 
     bool exists(const evmc::address& address) const noexcept;
 
@@ -106,7 +106,7 @@ class IntraBlockState {
 
     uint64_t total_refund() const noexcept;
 
-   private:
+  private:
     friend class state::CreateDelta;
     friend class state::UpdateDelta;
     friend class state::SuicideDelta;
@@ -121,7 +121,7 @@ class IntraBlockState {
 
     void touch(const evmc::address& address) noexcept;
 
-    db::StateBuffer& db_;
+    StateBuffer& db_;
 
     mutable absl::flat_hash_map<evmc::address, state::Object> objects_;
     mutable absl::flat_hash_map<evmc::address, state::Storage> storage_;