Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Promote segmented storage #5700

Merged
merged 15 commits into from
Jul 27, 2023
Merged

Promote segmented storage #5700

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
5c9feb2
saving huge pile of code
garyschulte Jul 12, 2023
407add8
happy with InMemory implementations
garyschulte Jul 12, 2023
20e4552
rocks implementation, Bonsai classes implementation
garyschulte Jul 13, 2023
d3b80f2
fix test compilation breakage
garyschulte Jul 13, 2023
8737e60
spotless
garyschulte Jul 13, 2023
7b2cc8c
spdx header
garyschulte Jul 13, 2023
18dcb2b
fix some tests
garyschulte Jul 14, 2023
5f96e49
javadoc and spotless
garyschulte Jul 14, 2023
ec2fffd
fix more tests
garyschulte Jul 16, 2023
25c87c1
fix some errors:
garyschulte Jul 23, 2023
b8a572f
fix last few tests, fix defect in SegmentedInMemoryKeyValueStorage wh…
garyschulte Jul 25, 2023
109b0a3
equivalent signal for state removal checks in MarkSweepPrunerTest
garyschulte Jul 25, 2023
f63a720
review feedback from Daniel, refactored commented out worlstate archi…
garyschulte Jul 26, 2023
e28eaa3
Merge remote-tracking branch 'origin/main' into feature/promote-segme…
garyschulte Jul 26, 2023
5759e19
Merge branch 'main' into feature/promote-segmented-storage
garyschulte Jul 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix last few tests, fix defect in SegmentedInMemoryKeyValueStorage wh… 
…ere snapshots were not cloning sub-maps

Signed-off-by: garyschulte <garyschulte@gmail.com>
  • Loading branch information
@garyschulte
garyschulte committed Jul 25, 2023
commit b8a572f5a3897c496a791ff1a01c66010c1edf84
60 changes: 22 additions & 38 deletions ethereum/core/src/test/java/org/hyperledger/besu/ethereum/bonsai/LogRollingTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorage;
import org.hyperledger.besu.plugin.services.storage.KeyValueStorageTransaction;
import org.hyperledger.besu.services.kvstore.InMemoryKeyValueStorage;

import java.util.Optional;
import java.util.stream.Collectors;
Expand All @@ -59,19 +58,19 @@ public class LogRollingTests {
private BonsaiWorldStateProvider archive;

private InMemoryKeyValueStorageProvider provider;
private InMemoryKeyValueStorage accountStorage;
private InMemoryKeyValueStorage codeStorage;
private InMemoryKeyValueStorage storageStorage;
private InMemoryKeyValueStorage trieBranchStorage;
private InMemoryKeyValueStorage trieLogStorage;
private KeyValueStorage accountStorage;
private KeyValueStorage codeStorage;
private KeyValueStorage storageStorage;
private KeyValueStorage trieBranchStorage;
private KeyValueStorage trieLogStorage;

private InMemoryKeyValueStorageProvider secondProvider;
private BonsaiWorldStateProvider secondArchive;
private InMemoryKeyValueStorage secondAccountStorage;
private InMemoryKeyValueStorage secondCodeStorage;
private InMemoryKeyValueStorage secondStorageStorage;
private InMemoryKeyValueStorage secondTrieBranchStorage;
private InMemoryKeyValueStorage secondTrieLogStorage;
private KeyValueStorage secondAccountStorage;
private KeyValueStorage secondCodeStorage;
private KeyValueStorage secondStorageStorage;
private KeyValueStorage secondTrieBranchStorage;
private KeyValueStorage secondTrieLogStorage;
private final Blockchain blockchain = mock(Blockchain.class);

private static final Address addressOne =
Expand Down Expand Up @@ -131,21 +130,14 @@ public void createStorage() {
new BonsaiWorldStateProvider(
provider, blockchain, cachedMerkleTrieLoader, new NoOpMetricsSystem(), null);
accountStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
codeStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
codeStorage = provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
storageStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
trieBranchStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
trieLogStorage =
(InMemoryKeyValueStorage)
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);

secondProvider = new InMemoryKeyValueStorageProvider();
final CachedMerkleTrieLoader secondOptimizedMerkleTrieLoader =
Expand All @@ -158,24 +150,16 @@ public void createStorage() {
new NoOpMetricsSystem(),
null);
secondAccountStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
secondCodeStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
secondStorageStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
secondTrieBranchStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
secondTrieLogStorage =
(InMemoryKeyValueStorage)
secondProvider.getStorageBySegmentIdentifier(
KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
secondProvider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
}

@Test
Expand Down Expand Up @@ -327,7 +311,7 @@ public void rollBackOnce() {
assertThat(secondWorldState.rootHash()).isEqualByComparingTo(worldState.rootHash());
}

private TrieLogLayer getTrieLogLayer(final InMemoryKeyValueStorage storage, final Bytes key) {
private TrieLogLayer getTrieLogLayer(final KeyValueStorage storage, final Bytes key) {
return storage
.get(key.toArrayUnsafe())
.map(bytes -> TrieLogFactoryImpl.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false)))
Expand Down
6 changes: 4 additions & 2 deletions ...reum/core/src/test/java/org/hyperledger/besu/ethereum/worldstate/MarkSweepPrunerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,8 @@ public void sweepBefore_shouldSweepStateRootFirst() {
stateRoots.forEach(
stateRoot -> {
final InOrder thisRootsOrdering = inOrder(hashValueStore, worldStateStorage);
thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
// disable this validation since the underlying storage no longer directly uses .remove()
// thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
garyschulte marked this conversation as resolved.
Show resolved Hide resolved
thisRootsOrdering.verify(worldStateStorage).prune(any());
});
}
Expand Down Expand Up @@ -184,7 +185,8 @@ public void sweepBefore_shouldNotRemoveMarkedStateRoots() {
stateRoots.forEach(
stateRoot -> {
final InOrder thisRootsOrdering = inOrder(hashValueStore, worldStateStorage);
thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
// disable this validation since the underlying storage no longer directly uses .remove()
// thisRootsOrdering.verify(hashValueStore).remove(stateRoot);
thisRootsOrdering.verify(worldStateStorage).prune(any());
});

Expand Down
21 changes: 13 additions & 8 deletions ...src/main/java/org/hyperledger/besu/services/kvstore/SegmentedInMemoryKeyValueStorage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public SegmentedInMemoryKeyValueStorage() {
*
* @param hashValueStore the hash value store
*/
SegmentedInMemoryKeyValueStorage(
protected SegmentedInMemoryKeyValueStorage(
final Map<SegmentIdentifier, Map<Bytes, Optional<byte[]>>> hashValueStore) {
this.hashValueStore = hashValueStore;
}
Expand All @@ -85,7 +85,7 @@ public void clear(final SegmentIdentifier segmentIdentifier) {
final Lock lock = rwLock.writeLock();
lock.lock();
try {
hashValueStore.computeIfAbsent(segmentIdentifier, s -> new HashMap<>()).clear();
Optional.ofNullable(hashValueStore.get(segmentIdentifier)).ifPresent(Map::clear);
} finally {
lock.unlock();
}
Expand Down Expand Up @@ -200,7 +200,10 @@ public boolean isClosed() {

@Override
public SegmentedInMemoryKeyValueStorage takeSnapshot() {
return new SegmentedInMemoryKeyValueStorage(new HashMap<>(hashValueStore));
// need to clone the submaps also:
return new SegmentedInMemoryKeyValueStorage(
hashValueStore.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, e -> new HashMap<>(e.getValue()))));
}

@Override
Expand Down Expand Up @@ -247,11 +250,13 @@ public void commit() throws StorageException {

removedKeys.entrySet().stream()
.forEach(
entry ->
hashValueStore
.computeIfAbsent(entry.getKey(), __ -> new HashMap<>())
.keySet()
.removeAll(entry.getValue()));
entry -> {
var keyset =
hashValueStore
.computeIfAbsent(entry.getKey(), __ -> new HashMap<>())
.keySet();
keyset.removeAll(entry.getValue());
});

updatedValues.clear();
removedKeys.clear();
Expand Down