feat(sdk): get identity by non-unique pubkey hashes

chore: update to latest dash core 37 (#2483)

feat(platform)!: token advanced distribution and updates (#2471)

fix: token history contract (#2474)

Co-authored-by: Ivan Shumkov <ivan@shumkov.ru>
Co-authored-by: QuantumExplorer <quantum@dash.org>

fix(drive): using new rust dash core methods for reversed quorum hash to maintain backwards compatibility (#2489)

feat: more granular integer document property types (#2455)

Co-authored-by: Quantum Explorer <quantum@dash.org>

docs: update comment for data contract code range (#2476)

feat: validate token name localizations (#2468)

feat(sdk): get identity by non-unique keys

build(deps): update grovedb to current develop

test: test identity by non-unique pubkey hashes

fix(sdk): dash core client fails to get quorum

chore: minor fixes

test(drive-abci): identity by non-unique pubkey start after

chore: minor changes to verify

feat(sdk): token and group queries (#2449)

chore: revert limit 1 => limit none

chore: add non-unique key to test identities

test(sdk): test vectors for test_fetch_identity_by_non_unique_public_keys

fix(platform)!: token distribution fixes and tests (#2494)

chore(platform): bump to version 2.0.0-dev.1 (#2495)

test: update assertion

fix(sdk): make some things public (#2496)

feat(platform): require token for document actions (#2498)

fix: data contract proof doesn't work  with new auto fields (#2501)

Author: lklimek

packages/dapi-grpc/build.rs

@@ -63,7 +63,7 @@ fn configure_platform(mut platform: MappingConfig) -> MappingConfig {
     // Derive features for versioned messages
     //
     // "GetConsensusParamsRequest" is excluded as this message does not support proofs
-    const VERSIONED_REQUESTS: [&str; 40] = [
+    const VERSIONED_REQUESTS: [&str; 41] = [
         "GetDataContractHistoryRequest",
         "GetDataContractRequest",
         "GetDataContractsRequest",
@@ -75,6 +75,7 @@ fn configure_platform(mut platform: MappingConfig) -> MappingConfig {
         "GetIdentityContractNonceRequest",
         "GetIdentityBalanceAndRevisionRequest",
         "GetIdentityBalanceRequest",
+        "GetIdentityByNonUniquePublicKeyHashRequest",
         "GetIdentityByPublicKeyHashRequest",
         "GetIdentityKeysRequest",
         "GetIdentityRequest",
@@ -110,6 +111,9 @@ fn configure_platform(mut platform: MappingConfig) -> MappingConfig {
     // - "GetConsensusParamsResponse"
     // - "GetStatusResponse"
     //
+    // The following responses are excluded as they need custom proof handling:
+    // - "GetIdentityByNonUniquePublicKeyHashResponse"
+    //
     //  "GetEvonodesProposedEpochBlocksResponse" is used for 2 Requests
     const VERSIONED_RESPONSES: [&str; 39] = [
         "GetDataContractHistoryResponse",

packages/rs-dapi-client/src/transport/grpc.rs

@@ -490,6 +490,15 @@ impl_transport_request_grpc!(
     get_status
 );
 
+//   rpc getIdentityByNonUniquePublicKeyHash(GetIdentityByNonUniquePublicKeyHashRequest) returns (GetIdentityByNonUniquePublicKeyHashResponse);
+impl_transport_request_grpc!(
+    platform_proto::GetIdentityByNonUniquePublicKeyHashRequest,
+    platform_proto::GetIdentityByNonUniquePublicKeyHashResponse,
+    PlatformGrpcClient,
+    RequestSettings::default(),
+    get_identity_by_non_unique_public_key_hash
+);
+
 // rpc getIdentityTokenBalances(GetIdentityTokenBalancesRequest) returns (GetIdentityTokenBalancesResponse);
 impl_transport_request_grpc!(
     platform_proto::GetIdentityTokenBalancesRequest,

packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/tokens.rs

@@ -110,14 +110,20 @@ impl<C> Platform<C> {
         transaction: TransactionArg,
         platform_version: &PlatformVersion,
     ) -> Result<(), Error> {
+        let mut rng = StdRng::seed_from_u64(0u64);
+        let non_unique_key =
+            IdentityPublicKey::random_voting_key_with_rng(11, &mut rng, platform_version)?;
+
         for id in [IDENTITY_ID_1, IDENTITY_ID_2, IDENTITY_ID_3] {
             // Create identity without keys
             let mut identity = Identity::create_basic_identity(id, platform_version)?;
 
             // Generate keys
             let seed = id.to_buffer()[0];
             let mut rng = StdRng::seed_from_u64(seed as u64);
-            let keys = IdentityPublicKey::main_keys_with_random_authentication_keys_with_private_keys_with_rng(3, &mut rng, platform_version)?;
+            let mut keys = IdentityPublicKey::main_keys_with_random_authentication_keys_with_private_keys_with_rng(3, &mut rng, platform_version)?;
+            // every identity has the same non-unique key
+            keys.push(non_unique_key.clone());
 
             for (key, private_key) in keys.iter() {
                 let private_key = hex::encode(private_key);

packages/rs-drive-abci/src/query/identity_based_queries/identity_by_non_unique_public_key_hash/mod.rs

@@ -7,7 +7,6 @@ use dapi_grpc::platform::v0::get_identity_by_non_unique_public_key_hash_request:
 use dapi_grpc::platform::v0::get_identity_by_non_unique_public_key_hash_response::Version as ResponseVersion;
 use dapi_grpc::platform::v0::{
     GetIdentityByNonUniquePublicKeyHashRequest, GetIdentityByNonUniquePublicKeyHashResponse,
-    GetIdentityByPublicKeyHashResponse,
 };
 use dpp::version::PlatformVersion;
 
@@ -28,7 +27,8 @@ impl<C> Platform<C> {
                 ),
             ));
         };
-
+        // TODO why `identity_by_unique_public_key_hash`?
+        // Shouldn't we rename or add new field like `identity_by_non_unique_public_key_hash`?
         let feature_version_bounds = &platform_version
             .drive_abci
             .query

packages/rs-drive-abci/src/query/identity_based_queries/identity_by_non_unique_public_key_hash/v0/mod.rs

@@ -38,7 +38,7 @@ impl<C> Platform<C> {
             )
             .map(|bytes| bytes.0)
             .map_err(|_| QueryError::InvalidArgument(
-                "public key hash must be 20 bytes long".to_string()
+                "start_after must be 32 bytes long identity ID".to_string()
             ))))
         } else {
             None
@@ -129,6 +129,33 @@ mod tests {
         ));
     }
 
+    #[test]
+    fn test_invalid_start_after() {
+        let (platform, state, version) = setup_platform(None, Network::Testnet, None);
+
+        let negative_tests: Vec<&[u8]> = vec![&[0u8; 4], &[0u8; 20], &[0u8; 64]];
+
+        for test in negative_tests {
+            let request = GetIdentityByNonUniquePublicKeyHashRequestV0 {
+                public_key_hash: vec![0; 20],
+                start_after: Some(test.to_vec()),
+                prove: false,
+            };
+
+            let result = platform
+                .query_identity_by_non_unique_public_key_hash_v0(request, &state, version)
+                .expect("expected query to succeed");
+
+            assert!(
+                matches!(
+                result.errors.as_slice(),
+                [QueryError::InvalidArgument(msg)] if msg == &"start_after must be 32 bytes long identity ID".to_string()),
+                "errors: {:?}",
+                result.errors,
+            );
+        }
+    }
+
     #[test]
     fn test_identity_not_found() {
         let (platform, state, version) = setup_platform(None, Network::Testnet, None);

packages/rs-drive-proof-verifier/src/proof.rs

@@ -15,7 +15,10 @@ use dapi_grpc::platform::v0::get_protocol_version_upgrade_vote_status_request::{
     self, GetProtocolVersionUpgradeVoteStatusRequestV0,
 };
 use dapi_grpc::platform::v0::security_level_map::KeyKindRequestType as GrpcKeyKind;
-use dapi_grpc::platform::v0::{get_contested_resource_identity_votes_request, get_data_contract_history_request, get_data_contract_request, get_data_contracts_request, get_epochs_info_request, get_evonodes_proposed_epoch_blocks_by_ids_request, get_evonodes_proposed_epoch_blocks_by_range_request, get_group_actions_request, get_group_info_request, get_group_infos_request, get_identities_balances_request, get_identities_contract_keys_request, get_identity_balance_and_revision_request, get_identity_balance_request, get_identity_by_public_key_hash_request, get_identity_contract_nonce_request, get_identity_keys_request, get_identity_nonce_request, get_identity_request, get_path_elements_request, get_prefunded_specialized_balance_request, GetContestedResourceVotersForIdentityRequest, GetContestedResourceVotersForIdentityResponse, GetGroupActionSignersRequest, GetGroupActionSignersResponse, GetGroupActionsRequest, GetGroupActionsResponse, GetGroupInfoRequest, GetGroupInfoResponse, GetGroupInfosRequest, GetGroupInfosResponse, GetPathElementsRequest, GetPathElementsResponse, GetProtocolVersionUpgradeStateRequest, GetProtocolVersionUpgradeStateResponse, GetProtocolVersionUpgradeVoteStatusRequest, GetProtocolVersionUpgradeVoteStatusResponse, Proof, ResponseMetadata};
+use dapi_grpc::platform::v0::{
+    get_contested_resource_identity_votes_request, get_data_contract_history_request, get_data_contract_request, get_data_contracts_request, get_epochs_info_request, get_evonodes_proposed_epoch_blocks_by_ids_request, get_evonodes_proposed_epoch_blocks_by_range_request, get_identities_balances_request, get_identities_contract_keys_request, get_identity_balance_and_revision_request, get_identity_balance_request, get_identity_by_non_unique_public_key_hash_request,
+    get_identity_by_public_key_hash_request, get_identity_contract_nonce_request, get_identity_keys_request, get_identity_nonce_request, get_identity_request, get_path_elements_request, get_prefunded_specialized_balance_request, GetContestedResourceVotersForIdentityRequest, GetContestedResourceVotersForIdentityResponse, GetPathElementsRequest, GetPathElementsResponse, GetProtocolVersionUpgradeStateRequest, GetProtocolVersionUpgradeStateResponse, GetProtocolVersionUpgradeVoteStatusRequest, GetProtocolVersionUpgradeVoteStatusResponse, Proof, ResponseMetadata
+};
 use dapi_grpc::platform::{
     v0::{self as platform, key_request_type, KeyRequestType as GrpcKeyType},
     VersionedGrpcResponse,
@@ -36,6 +39,7 @@ use dpp::state_transition::proof_result::StateTransitionProofResult;
 use dpp::state_transition::StateTransition;
 use dpp::version::PlatformVersion;
 use dpp::voting::votes::Vote;
+use drive::drive::identity::identity_and_non_unique_public_key_hash_double_proof::IdentityAndNonUniquePublicKeyHashDoubleProof;
 use drive::drive::identity::key::fetch::{
     IdentityKeysRequest, KeyKindRequestType, KeyRequestType, PurposeU8, SecurityLevelU8,
 };
@@ -339,6 +343,98 @@ impl FromProof<platform::GetIdentityByPublicKeyHashRequest> for Identity {
     }
 }
 
+impl FromProof<platform::GetIdentityByNonUniquePublicKeyHashRequest> for Identity {
+    type Request = platform::GetIdentityByNonUniquePublicKeyHashRequest;
+    type Response = platform::GetIdentityByNonUniquePublicKeyHashResponse;
+    fn maybe_from_proof_with_metadata<'a, I: Into<Self::Request>, O: Into<Self::Response>>(
+        request: I,
+        response: O,
+        _network: Network,
+        platform_version: &PlatformVersion,
+        provider: &'a dyn ContextProvider,
+    ) -> Result<(Option<Self>, ResponseMetadata, Proof), Error>
+    where
+        Self: Sized + 'a,
+    {
+        let request = request.into();
+        let response = response.into();
+        // Parse response to read proof and metadata
+        // note that proof in this case is different
+        // let proof = response.proof().or(Err(Error::NoProofInResult))?;
+        use platform::get_identity_by_non_unique_public_key_hash_response::{
+            get_identity_by_non_unique_public_key_hash_response_v0::Result as V0Result, Version::V0,
+        };
+
+        let (proved_response, mtd) = match response.version {
+            Some(V0(v0)) => {
+                let proof = if let V0Result::Proof(p) = v0.result.ok_or(Error::NoProofInResult)? {
+                    p
+                } else {
+                    return Err(Error::NoProofInResult);
+                };
+
+                (proof, v0.metadata.ok_or(Error::EmptyResponseMetadata)?)
+            }
+            _ => return Err(Error::EmptyResponseMetadata),
+        };
+
+        // let mtd = response.metadata().or(Err(Error::EmptyResponseMetadata))?;
+
+        let (public_key_hash, after_identity) = match request.version.ok_or(Error::EmptyVersion)? {
+            get_identity_by_non_unique_public_key_hash_request::Version::V0(v0) => {
+                let public_key_hash =
+                    v0.public_key_hash
+                        .try_into()
+                        .map_err(|_| Error::RequestError {
+                            error: "Invalid public key hash length".to_string(),
+                        })?;
+
+                let after = v0
+                    .start_after
+                    .map(|a| {
+                        a.try_into().map_err(|_| Error::RequestError {
+                            error: "Invalid start_after length".to_string(),
+                        })
+                    })
+                    .transpose()?;
+                (public_key_hash, after)
+            }
+        };
+
+        // we need to convert some data to handle non-default proof structure for this response
+        let proof = proved_response
+            .grovedb_identity_public_key_hash_proof
+            .ok_or(Error::NoProofInResult)?;
+
+        let proof_tuple = IdentityAndNonUniquePublicKeyHashDoubleProof {
+            identity_proof: proved_response.identity_proof_bytes,
+            identity_id_public_key_hash_proof: proof.grovedb_proof.clone(),
+        };
+
+        // Extract content from proof and verify Drive/GroveDB proofs
+        let (root_hash, maybe_identity) =
+            Drive::verify_full_identity_by_non_unique_public_key_hash(
+                &proof_tuple,
+                public_key_hash,
+                after_identity,
+                platform_version,
+            )
+            .map_err(|e| match e {
+                drive::error::Error::GroveDB(e) => Error::GroveDBError {
+                    proof_bytes: proof.grovedb_proof.clone(),
+                    height: mtd.height,
+                    time_ms: mtd.time_ms,
+                    error: e.to_string(),
+                },
+                _ => e.into(),
+            })?;
+
+        verify_tenderdash_proof(&proof, &mtd, &root_hash, provider)?;
+
+        Ok((maybe_identity, mtd.clone(), proof))
+    }
+}
+
 impl FromProof<platform::GetIdentityKeysRequest> for IdentityPublicKeys {
     type Request = platform::GetIdentityKeysRequest;
     type Response = platform::GetIdentityKeysResponse;

packages/rs-drive-verify-c-binding/src/lib.rs

@@ -100,7 +100,7 @@ pub unsafe extern "C" fn verify_full_identity_by_identity_id(
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn verify_identity_id_by_public_key_hash(
+pub unsafe extern "C" fn verify_identity_id_by_unique_public_key_hash(
     proof_array: *const u8,
     proof_len: usize,
     is_proof_subset: bool,
@@ -109,8 +109,11 @@ pub unsafe extern "C" fn verify_identity_id_by_public_key_hash(
     let proof = unsafe { slice::from_raw_parts(proof_array, proof_len) };
     let public_key_hash = unsafe { std::ptr::read(public_key_hash) };
 
-    let verification_result =
-        Drive::verify_identity_id_by_public_key_hash(proof, is_proof_subset, public_key_hash);
+    let verification_result = Drive::verify_identity_id_by_unique_public_key_hash(
+        proof,
+        is_proof_subset,
+        public_key_hash,
+    );
 
     match verification_result {
         Ok((root_hash, maybe_identity_id)) => {
@@ -680,13 +683,13 @@ mod tests {
     }
 
     #[test]
-    fn verify_identity_id_by_public_key_hash() {
+    fn verify_identity_id_by_unique_public_key_hash() {
         let proof = multiple_identity_proof();
         let public_key_hash: PublicKeyHash = [
             31, 8, 21, 38, 154, 252, 1, 45, 228, 66, 96, 206, 178, 138, 68, 150, 211, 24, 65, 132,
         ];
         let (_root_hash, maybe_identity_id) =
-            Drive::verify_identity_id_by_public_key_hash(proof, true, public_key_hash)
+            Drive::verify_identity_id_by_unique_public_key_hash(proof, true, public_key_hash)
                 .expect("should verify");
         let expected_identity_id: [u8; 32] = [
             15, 126, 159, 152, 150, 254, 206, 186, 180, 193, 157, 65, 233, 215, 241, 108, 23, 39,

packages/rs-drive/Cargo.toml

@@ -52,12 +52,12 @@ enum-map = { version = "2.0.3", optional = true }
 intmap = { version = "3.0.1", features = ["serde"], optional = true }
 chrono = { version = "0.4.35", optional = true }
 itertools = { version = "0.13", optional = true }
-grovedb = { git = "https://github.com/dashpay/grovedb", rev= "f89e03e4e0ac12aa2feea5c94b38c09f4909facc", optional = true, default-features = false }
-grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev= "f89e03e4e0ac12aa2feea5c94b38c09f4909facc", optional = true }
-grovedb-path = { git = "https://github.com/dashpay/grovedb", rev= "f89e03e4e0ac12aa2feea5c94b38c09f4909facc" }
-grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev= "f89e03e4e0ac12aa2feea5c94b38c09f4909facc", optional = true }
-grovedb-version = { git = "https://github.com/dashpay/grovedb", rev= "f89e03e4e0ac12aa2feea5c94b38c09f4909facc" }
-grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev= "f89e03e4e0ac12aa2feea5c94b38c09f4909facc" }
+grovedb = { git = "https://github.com/dashpay/grovedb", rev = "f89e03e4e0ac12aa2feea5c94b38c09f4909facc", optional = true, default-features = false }
+grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "f89e03e4e0ac12aa2feea5c94b38c09f4909facc", optional = true }
+grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "f89e03e4e0ac12aa2feea5c94b38c09f4909facc" }
+grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "f89e03e4e0ac12aa2feea5c94b38c09f4909facc", optional = true }
+grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "f89e03e4e0ac12aa2feea5c94b38c09f4909facc" }
+grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "f89e03e4e0ac12aa2feea5c94b38c09f4909facc" }
 
 [dev-dependencies]
 criterion = "0.5"
@@ -85,6 +85,7 @@ name = "benchmarks"
 harness = false
 
 [features]
+
 default = ["full", "verify", "fixtures-and-mocks", "cbor_query"]
 grovedbg = ["grovedb/grovedbg"]
 fee-distribution = ["dpp/fee-distribution"]
@@ -117,4 +118,9 @@ server = [
 full = ["server", "ciborium", "serde", "bs58", "tempfile", "base64", "chrono"]
 cbor_query = ["ciborium", "dpp/platform-value-cbor", "dpp/cbor"]
 grovedb_operations_logging = []
-verify = ["grovedb/verify", "grovedb-costs", "dpp/state-transitions", "dpp/system_contracts"]
+verify = [
+  "grovedb/verify",
+  "grovedb-costs",
+  "dpp/state-transitions",
+  "dpp/system_contracts",
+]

packages/rs-drive/src/drive/identity/fetch/queries/mod.rs

@@ -1,9 +1,7 @@
 use crate::drive::balances::balance_path_vec;
 use crate::drive::identity::key::fetch::IdentityKeysRequest;
-use crate::drive::{
-    identity_tree_path_vec, non_unique_key_hashes_tree_path, non_unique_key_hashes_tree_path_vec,
-    unique_key_hashes_tree_path_vec, Drive,
-};
+use crate::drive::non_unique_key_hashes_tree_path_vec;
+use crate::drive::{identity_tree_path_vec, unique_key_hashes_tree_path_vec, Drive};
 use std::ops::RangeFull;
 
 use crate::error::Error;
@@ -102,8 +100,9 @@ impl Drive {
         let non_unique_key_hashes = non_unique_key_hashes_tree_path_vec();
         let mut query = Query::new_single_key(public_key_hash.to_vec());
         let sub_query = if let Some(after) = after {
-            Query::new_single_query_item(QueryItem::RangeFrom(after.to_vec()..))
+            Query::new_single_query_item(QueryItem::RangeAfter(after.to_vec()..))
         } else {
+            // TODO: why not limit 1?
             Query::new_range_full()
         };
         query.set_subquery(sub_query);

packages/rs-drive/src/drive/mod.rs

@@ -261,7 +261,7 @@ pub(crate) fn non_unique_key_hashes_tree_path() -> [&'static [u8]; 1] {
 }
 
 /// Returns the path to the masternode key hashes.
-#[cfg(feature = "server")]
+#[cfg(any(feature = "server", feature = "verify"))]
 pub(crate) fn non_unique_key_hashes_tree_path_vec() -> Vec<Vec<u8>> {
     vec![vec![
         RootTree::NonUniquePublicKeyKeyHashesToIdentities as u8,
@@ -278,7 +278,7 @@ pub(crate) fn non_unique_key_hashes_sub_tree_path(public_key_hash: &[u8]) -> [&[
 }
 
 /// Returns the path to the masternode key hashes sub tree.
-#[cfg(feature = "server")]
+#[cfg(any(feature = "server", feature = "verify"))]
 pub(crate) fn non_unique_key_hashes_sub_tree_path_vec(public_key_hash: [u8; 20]) -> Vec<Vec<u8>> {
     vec![
         vec![RootTree::NonUniquePublicKeyKeyHashesToIdentities as u8],