Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create new trait for non-dedup storage decode #1932

Merged
merged 49 commits into from
Nov 2, 2023
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
29436cd
adjust test
0xmovses Oct 16, 2023
684e2eb
create new trait, add new test
0xmovses Oct 18, 2023
82415ec
Update substrate/frame/support/src/storage/mod.rs
0xmovses Oct 18, 2023
755922e
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 18, 2023
55f3f1e
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 19, 2023
adaf208
intergrate ggwpez feedback
0xmovses Oct 19, 2023
765002a
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 19, 2023
6be51ec
Update substrate/frame/support/src/storage/mod.rs
0xmovses Oct 19, 2023
4604f56
Update substrate/frame/support/src/storage/mod.rs
0xmovses Oct 19, 2023
8d92cd9
rename methods
0xmovses Oct 19, 2023
f0bfceb
remove redundant impl
0xmovses Oct 19, 2023
02e7cd5
improve documentaion
0xmovses Oct 19, 2023
bc2c39e
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 19, 2023
9a812a4
fmt
0xmovses Oct 23, 2023
e7e702b
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 23, 2023
628f00b
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 23, 2023
3216351
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 23, 2023
8c0b712
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 23, 2023
42f09d1
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 23, 2023
cab8b41
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 23, 2023
1693d50
add blanket impl
0xmovses Oct 23, 2023
a3f706c
fmt
0xmovses Oct 23, 2023
14dd23f
merge Cargo.lock master
0xmovses Oct 23, 2023
16dcd96
clippy
0xmovses Oct 23, 2023
a2ba001
stop doc example from running
0xmovses Oct 24, 2023
1fd8567
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 24, 2023
0aa1802
Update substrate/frame/support/src/storage/mod.rs
0xmovses Oct 25, 2023
4740ff1
address ggwpez feedback
0xmovses Oct 25, 2023
07cdf1c
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 25, 2023
598d312
clippy
0xmovses Oct 25, 2023
b0e0019
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 25, 2023
2b541cb
undo fmt
0xmovses Oct 25, 2023
2c6ef28
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 30, 2023
63fb99d
address gupnik feedback
0xmovses Oct 30, 2023
d4a9626
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 30, 2023
3d2947b
update unit test
0xmovses Oct 30, 2023
4755967
fix doc/comment line wrap
0xmovses Oct 31, 2023
f7816b4
remove unused
0xmovses Oct 31, 2023
99efb5a
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 31, 2023
8731067
".git/.scripts/commands/fmt/fmt.sh"
Oct 31, 2023
e1143ac
remove unused
0xmovses Oct 31, 2023
cce11ef
Merge branch 'master' into 0xmovses-storage-length
0xmovses Oct 31, 2023
37c0802
Merge branch 'master' into 0xmovses-storage-length
0xmovses Nov 1, 2023
1cd08d1
add new unit test, add new trait to underlying types
0xmovses Nov 1, 2023
c0b8a13
Merge branch 'master' into 0xmovses-storage-length
0xmovses Nov 1, 2023
d13fabb
".git/.scripts/commands/fmt/fmt.sh"
Nov 1, 2023
58b4622
Merge branch 'master' into 0xmovses-storage-length
0xmovses Nov 1, 2023
ce4e9f3
Merge branch 'master' into 0xmovses-storage-length
0xmovses Nov 1, 2023
cbea9c5
Merge branch 'master' into 0xmovses-storage-length
0xmovses Nov 2, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion substrate/frame/support/src/storage/bounded_btree_set.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

//! Traits, types and structs to support a bounded `BTreeSet`.

use crate::storage::StorageDecodeLength;
use frame_support::storage::StorageDecodeLength;
pub use sp_runtime::BoundedBTreeSet;

impl<T, S> StorageDecodeLength for BoundedBTreeSet<T, S> {}
0xmovses marked this conversation as resolved.
Show resolved Hide resolved
Expand Down
80 changes: 76 additions & 4 deletions substrate/frame/support/src/storage/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,32 @@ pub trait StorageValue<T: FullCodec> {
{
T::decode_len(&Self::hashed_key())
}

/// Read the length of the storage value without decoding the entire value.
///
/// `T` is required to implement [`StorageDecodeNonDedupLength`].
///
/// If the value does not exists or it fails to decode the length, `None` is returned.
/// Otherwise `Some(len)` is returned.
///
/// # Warning
///
/// - The value returned is the non-deduplicated length of the underlying Vector in storage.This
/// means that any duplicate items are included.
///
/// - `None` does not mean that `get()` does not return a value. The default value is completely
/// ignored by this function.
///
/// # Example
#[doc = docify::embed!("src/storage/mod.rs", btree_set_decode_non_dedup_len)]
0xmovses marked this conversation as resolved.
Show resolved Hide resolved
/// This demonstrates how `decode_non_dedup_len` will count even the duplicate values in the
/// storage (in this case, the number `4` is counted twice).
fn decode_non_dedup_len() -> Option<usize>
where
T: StorageDecodeNonDedupLength,
{
T::decode_non_dedup_len(&Self::hashed_key())
}
}

/// A non-continuous container type.
Expand Down Expand Up @@ -1400,8 +1426,7 @@ pub trait StoragePrefixedMap<Value: FullCodec> {
/// This trait is sealed.
pub trait StorageAppend<Item: Encode>: private::Sealed {}

/// Marker trait that will be implemented for types that support to decode their length in an
kianenigma marked this conversation as resolved.
Show resolved Hide resolved
/// efficient way. It is expected that the length is at the beginning of the encoded object
/// It is expected that the length is at the beginning of the encoded object
/// and that the length is a `Compact<u32>`.
///
/// This trait is sealed.
Expand All @@ -1421,6 +1446,29 @@ pub trait StorageDecodeLength: private::Sealed + codec::DecodeLength {
}
}

/// It is expected that the length is at the beginning of the encoded objectand that the length is a
/// `Compact<u32>`.
///
/// # Note
/// The length returned by this trait is not deduplicated, i.e. it is the length of the underlying
/// stored Vec.
///
/// This trait is sealed.
pub trait StorageDecodeNonDedupLength: private::Sealed + codec::DecodeLength {
/// Decode the length of the storage value at `key`.
///
/// This function assumes that the length is at the beginning of the encoded object and is a
/// `Compact<u32>`.
///
/// Returns `None` if the storage value does not exist or the decoding failed.
fn decode_non_dedup_len(key: &[u8]) -> Option<usize> {
let mut data = [0u8; 5];
let len = sp_io::storage::read(key, &mut data, 0)?;
let len = data.len().min(len as usize);
<Self as codec::DecodeLength>::len(&data[..len]).ok()
}
}

/// Provides `Sealed` trait to prevent implementing trait `StorageAppend` & `StorageDecodeLength`
/// & `EncodeLikeTuple` outside of this crate.
mod private {
Expand Down Expand Up @@ -1471,7 +1519,14 @@ impl<T: Encode> StorageAppend<T> for Vec<T> {}
impl<T: Encode> StorageDecodeLength for Vec<T> {}

impl<T: Encode> StorageAppend<T> for BTreeSet<T> {}
impl<T: Encode> StorageDecodeLength for BTreeSet<T> {}
impl<T: Encode> StorageDecodeNonDedupLength for BTreeSet<T> {}

// Blanket implementation StorageDecodeNonDedupLength for all types that are StorageDecodeLength.
impl<T: StorageDecodeLength> StorageDecodeNonDedupLength for T {
fn decode_non_dedup_len(key: &[u8]) -> Option<usize> {
T::decode_len(key)
}
}

/// We abuse the fact that SCALE does not put any marker into the encoding, i.e. we only encode the
/// internal vec and we can append to this vec. We have a test that ensures that if the `Digest`
Expand Down Expand Up @@ -2026,7 +2081,24 @@ mod test {
FooSet::append(6);
FooSet::append(7);

assert_eq!(FooSet::decode_len().unwrap(), 7);
assert_eq!(FooSet::decode_non_dedup_len().unwrap(), 7);
});
}

#[docify::export]
#[test]
fn btree_set_decode_non_dedup_len() {
#[crate::storage_alias]
type Store = StorageValue<Prefix, BTreeSet<u32>>;

TestExternalities::default().execute_with(|| {
Store::append(4);
Store::append(4); // duplicate value
Store::append(5);

let length_with_dup_items = 3;

assert_eq!(Store::decode_non_dedup_len().unwrap(), length_with_dup_items);
});
}
}