Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Promote segmented storage #5700

Merged
merged 15 commits into from
Jul 27, 2023
Merged

Promote segmented storage #5700

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
5c9feb2
saving huge pile of code
garyschulte Jul 12, 2023
407add8
happy with InMemory implementations
garyschulte Jul 12, 2023
20e4552
rocks implementation, Bonsai classes implementation
garyschulte Jul 13, 2023
d3b80f2
fix test compilation breakage
garyschulte Jul 13, 2023
8737e60
spotless
garyschulte Jul 13, 2023
7b2cc8c
spdx header
garyschulte Jul 13, 2023
18dcb2b
fix some tests
garyschulte Jul 14, 2023
5f96e49
javadoc and spotless
garyschulte Jul 14, 2023
ec2fffd
fix more tests
garyschulte Jul 16, 2023
25c87c1
fix some errors:
garyschulte Jul 23, 2023
b8a572f
fix last few tests, fix defect in SegmentedInMemoryKeyValueStorage wh…
garyschulte Jul 25, 2023
109b0a3
equivalent signal for state removal checks in MarkSweepPrunerTest
garyschulte Jul 25, 2023
f63a720
review feedback from Daniel, refactored commented out worlstate archi…
garyschulte Jul 26, 2023
e28eaa3
Merge remote-tracking branch 'origin/main' into feature/promote-segme…
garyschulte Jul 26, 2023
5759e19
Merge branch 'main' into feature/promote-segmented-storage
garyschulte Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rocks implementation, Bonsai classes implementation 
Signed-off-by: garyschulte <garyschulte@gmail.com>
  • Loading branch information
@garyschulte
garyschulte committed Jul 25, 2023
commit 20e45526b96824a45bca956737eb5bd614bf55b9
1 change: 0 additions & 1 deletion besu/src/test/java/org/hyperledger/besu/cli/CommandTestAbstract.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@
import org.hyperledger.besu.services.SecurityModuleServiceImpl;
import org.hyperledger.besu.services.StorageServiceImpl;
import org.hyperledger.besu.services.TransactionSelectionServiceImpl;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import java.io.ByteArrayOutputStream;
import java.io.File;
Expand Down
1 change: 0 additions & 1 deletion besu/src/test/java/org/hyperledger/besu/cli/subcommands/storage/StorageSubCommandTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@

import org.hyperledger.besu.cli.CommandTestAbstract;
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueSegmentIdentifier;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import org.junit.Test;
import org.junit.runner.RunWith;
Expand Down
1 change: 0 additions & 1 deletion besu/src/test/java/org/hyperledger/besu/controller/BesuControllerBuilderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.evm.internal.EvmConfiguration;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import java.math.BigInteger;
import java.time.Clock;
Expand Down
1 change: 0 additions & 1 deletion besu/src/test/java/org/hyperledger/besu/controller/MergeBesuControllerBuilderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.evm.internal.EvmConfiguration;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import java.math.BigInteger;
import java.time.Clock;
Expand Down
1 change: 0 additions & 1 deletion besu/src/test/java/org/hyperledger/besu/controller/QbftBesuControllerBuilderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import org.hyperledger.besu.ethereum.worldstate.WorldStateStorage;
import org.hyperledger.besu.evm.internal.EvmConfiguration;
import org.hyperledger.besu.metrics.ObservableMetricsSystem;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import java.math.BigInteger;
import java.time.Clock;
Expand Down
1 change: 1 addition & 0 deletions besu/src/test/java/org/hyperledger/besu/services/BesuEventsImplTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
import org.hyperledger.besu.plugin.data.PropagatedBlockContext;
import org.hyperledger.besu.plugin.data.SyncStatus;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
import org.hyperledger.besu.plugin.data.Transaction;
import org.hyperledger.besu.testutil.TestClock;

import java.math.BigInteger;
Expand Down
1 change: 0 additions & 1 deletion ...i/jsonrpc/websocket/subscription/blockheaders/NewBlockHeadersSubscriptionServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import org.hyperledger.besu.ethereum.storage.keyvalue.KeyValueStoragePrefixedKeyBlockchainStorage;
import org.hyperledger.besu.ethereum.storage.keyvalue.VariablesKeyValueStorage;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import java.util.List;
import java.util.function.Consumer;
Expand Down
1 change: 0 additions & 1 deletion ...integration-test/java/org/hyperledger/besu/ethereum/worldstate/PrunerIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import org.hyperledger.besu.ethereum.worldstate.Pruner.PruningPhase;
import org.hyperledger.besu.evm.worldstate.WorldState;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;
import org.hyperledger.besu.testutil.MockExecutorService;

import java.util.ArrayList;
Expand Down
4 changes: 2 additions & 2 deletions ...re/src/jmh/java/org/hyperledger/besu/ethereum/vm/operations/OperationBenchmarkHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBMetricsFactory;
import org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBConfigurationBuilder;
import org.hyperledger.besu.plugin.services.storage.rocksdb.segmented.OptimisticRocksDBColumnarKeyValueStorage;
import org.hyperledger.besu.services.kvstore.SnappableSegmentedKeyValueStorageAdapter;
import org.hyperledger.besu.services.kvstore.SegmentedKeyValueStorageAdapter;

import java.io.IOException;
import java.nio.file.Files;
Expand Down Expand Up @@ -70,7 +70,7 @@ public static OperationBenchmarkHelper create() throws IOException {
RocksDBMetricsFactory.PUBLIC_ROCKS_DB_METRICS);

final KeyValueStorage keyValueStorage =
new SnappableSegmentedKeyValueStorageAdapter<>(
new SegmentedKeyValueStorageAdapter(
KeyValueSegmentIdentifier.BLOCKCHAIN, optimisticRocksDBColumnarKeyValueStorage);

final ExecutionContextTestFixture executionContext =
Expand Down
25 changes: 5 additions & 20 deletions ...org/hyperledger/besu/ethereum/bonsai/storage/BonsaiSnapshotWorldStateKeyValueStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,19 +42,13 @@ public class BonsaiSnapshotWorldStateKeyValueStorage extends BonsaiWorldStateKey

public BonsaiSnapshotWorldStateKeyValueStorage(
final BonsaiWorldStateKeyValueStorage parentWorldStateStorage,
final SnappedKeyValueStorage accountStorage,
final SnappedKeyValueStorage codeStorage,
final SnappedKeyValueStorage storageStorage,
final SnappedKeyValueStorage trieBranchStorage,
final SnappedKeyValueStorage segmentedWorldStateStorage,
final KeyValueStorage trieLogStorage,
final ObservableMetricsSystem metricsSystem) {
super(
parentWorldStateStorage.flatDbMode,
parentWorldStateStorage.flatDbReaderStrategy,
accountStorage,
codeStorage,
storageStorage,
trieBranchStorage,
segmentedWorldStateStorage,
trieLogStorage,
metricsSystem);
this.parentWorldStateStorage = parentWorldStateStorage;
Expand All @@ -66,10 +60,7 @@ public BonsaiSnapshotWorldStateKeyValueStorage(
final ObservableMetricsSystem metricsSystem) {
this(
worldStateStorage,
((SnappableKeyValueStorage) worldStateStorage.accountStorage).takeSnapshot(),
((SnappableKeyValueStorage) worldStateStorage.codeStorage).takeSnapshot(),
((SnappableKeyValueStorage) worldStateStorage.storageStorage).takeSnapshot(),
((SnappableKeyValueStorage) worldStateStorage.trieBranchStorage).takeSnapshot(),
((SnappableKeyValueStorage) worldStateStorage.composedWorldStateStorage).takeSnapshot(),
worldStateStorage.trieLogStorage,
metricsSystem);
}
Expand All @@ -85,10 +76,7 @@ private boolean isClosedGet() {
@Override
public BonsaiUpdater updater() {
return new Updater(
((SnappedKeyValueStorage) accountStorage).getSnapshotTransaction(),
((SnappedKeyValueStorage) codeStorage).getSnapshotTransaction(),
((SnappedKeyValueStorage) storageStorage).getSnapshotTransaction(),
((SnappedKeyValueStorage) trieBranchStorage).getSnapshotTransaction(),
((SnappedKeyValueStorage) composedWorldStateStorage).getSnapshotTransaction(),
trieLogStorage.startTransaction());
}

Expand Down Expand Up @@ -223,10 +211,7 @@ protected synchronized void doClose() throws Exception {
subscribers.forEach(BonsaiStorageSubscriber::onCloseStorage);

// close all of the SnappedKeyValueStorages:
accountStorage.close();
codeStorage.close();
storageStorage.close();
trieBranchStorage.close();
composedWorldStateStorage.close();

// unsubscribe the parent worldstate
parentWorldStateStorage.unSubscribe(subscribeParentId);
Expand Down
Loading