Import pared-down version of parity-util-mem into mysten-infra (#134)

* Removed all custom allocator features, leaving only estimate-heapsize.
* Added MallocSizeOf implementaitons for some types used in narwhal.

Cargo.toml

@@ -3,6 +3,8 @@ members = [
     "crates/component",
     "crates/mysten-network",
     "crates/name-variant",
+    "crates/parity-util-mem",
+    "crates/parity-util-mem-derive",
     "crates/rccheck",
     "crates/telemetry-subscribers",
     "crates/typed-store",

crates/parity-util-mem-derive/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "parity-util-mem-derive"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+license = "MIT OR Apache-2.0"
+description = "Crate for memory reporting"
+repository = "https://github.com/paritytech/pariry-common/parity-util-mem/derive"
+edition = "2021"
+rust-version = "1.56.1"
+publish = false
+
+[lib]
+path = "lib.rs"
+proc-macro = true
+
+[dependencies]
+proc-macro2 = "1"
+syn = { version = "1", features = ["full"] }
+synstructure = "0.12"

crates/parity-util-mem-derive/lib.rs

@@ -0,0 +1,90 @@
+// Copyright (c) 2022, Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// Copyright 2020 Parity Technologies
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! A crate for deriving the MallocSizeOf trait.
+//!
+//! This is a copy of Servo malloc_size_of_derive code, modified to work with
+//! our `parity_util_mem` library
+
+#![allow(clippy::all)]
+
+extern crate proc_macro2;
+#[macro_use]
+extern crate syn;
+#[macro_use]
+extern crate synstructure;
+
+decl_derive!([MallocSizeOf, attributes(ignore_malloc_size_of)] => malloc_size_of_derive);
+
+fn malloc_size_of_derive(s: synstructure::Structure) -> proc_macro2::TokenStream {
+    let match_body = s.each(|binding| {
+        let ignore = binding
+            .ast()
+            .attrs
+            .iter()
+            .any(|attr| match attr.parse_meta().unwrap() {
+                syn::Meta::Path(ref path) | syn::Meta::List(syn::MetaList { ref path, .. })
+                    if path.is_ident("ignore_malloc_size_of") =>
+                {
+                    panic!(
+                        "#[ignore_malloc_size_of] should have an explanation, \
+					 e.g. #[ignore_malloc_size_of = \"because reasons\"]"
+                    );
+                }
+                syn::Meta::NameValue(syn::MetaNameValue { ref path, .. })
+                    if path.is_ident("ignore_malloc_size_of") =>
+                {
+                    true
+                }
+                _ => false,
+            });
+        if ignore {
+            None
+        } else if let syn::Type::Array(..) = binding.ast().ty {
+            Some(quote! {
+                for item in #binding.iter() {
+                    sum += parity_util_mem::MallocSizeOf::size_of(item, ops);
+                }
+            })
+        } else {
+            Some(quote! {
+                sum += parity_util_mem::MallocSizeOf::size_of(#binding, ops);
+            })
+        }
+    });
+
+    let ast = s.ast();
+    let name = &ast.ident;
+    let (impl_generics, ty_generics, where_clause) = ast.generics.split_for_impl();
+    let mut where_clause = where_clause.unwrap_or(&parse_quote!(where)).clone();
+    for param in ast.generics.type_params() {
+        let ident = &param.ident;
+        where_clause
+            .predicates
+            .push(parse_quote!(#ident: parity_util_mem::MallocSizeOf));
+    }
+
+    let tokens = quote! {
+        impl #impl_generics parity_util_mem::MallocSizeOf for #name #ty_generics #where_clause {
+            #[inline]
+            #[allow(unused_variables, unused_mut, unreachable_code)]
+            fn size_of(&self, ops: &mut parity_util_mem::MallocSizeOfOps) -> usize {
+                let mut sum = 0;
+                match *self {
+                    #match_body
+                }
+                sum
+            }
+        }
+    };
+
+    tokens
+}

crates/parity-util-mem/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "parity-util-mem"
+version = "0.11.0"
+authors = ["Parity Technologies <admin@parity.io>", "Andrew Schran <aschran@mystenlabs.com>"]
+repository = "https://github.com/paritytech/parity-common"
+description = "Collection of memory related utilities"
+license = "MIT OR Apache-2.0"
+edition = "2021"
+rust-version = "1.56.1"
+publish = false
+
+[dependencies]
+cfg-if = "1.0.0"
+lru = { version = "0.7", optional = true }
+hashbrown = { version = "0.12", optional = true }
+parity-util-mem-derive = { path = "../parity-util-mem-derive", version = "0.1" }
+impl-trait-for-tuples = "0.2.0"
+fastcrypto = { git = "https://github.com/MystenLabs/fastcrypto", rev = "c022a2ae23ca7cc2778293fd3b1db42e8cd02d3b", features = ["copy_key"] }
+indexmap = { version = "1.9.1", features = ["serde"] }
+
+smallvec = { version = "1.0.0", optional = true }
+parking_lot = { version = "0.12.0", optional = true }
+
+[features]
+default = ["std", "lru", "hashbrown", "smallvec", "estimate-heapsize"]
+std = ["parking_lot"]
+# Full estimate: no call to allocator
+estimate-heapsize = []

crates/parity-util-mem/src/allocators.rs

@@ -0,0 +1,96 @@
+// Copyright (c) 2022, Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+// Copyright 2020 Parity Technologies
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+#[cfg(feature = "std")]
+use crate::malloc_size::MallocUnconditionalSizeOf;
+use crate::malloc_size::{MallocSizeOf, MallocSizeOfOps, VoidPtrToSizeFn};
+#[cfg(not(feature = "std"))]
+use core::ffi::c_void;
+#[cfg(feature = "std")]
+use std::os::raw::c_void;
+
+mod usable_size {
+
+    use super::*;
+
+    cfg_if::cfg_if! {
+
+        if #[cfg(any(
+            target_arch = "wasm32",
+            feature = "estimate-heapsize",
+        ))] {
+
+            // do not try system allocator
+
+            /// Warning this is for compatibility only.
+            /// This function does panic: `estimate-heapsize` feature needs to be activated
+            /// to avoid this function call.
+            pub unsafe extern "C" fn malloc_usable_size(_ptr: *const c_void) -> usize {
+                unreachable!("estimate heapsize only")
+            }
+
+        } else if #[cfg(any(
+            target_os = "linux",
+            target_os = "android",
+            target_os = "freebsd",
+        ))] {
+            // Linux/BSD call system allocator (currently malloc).
+            extern "C" {
+                pub fn malloc_usable_size(ptr: *const c_void) -> usize;
+            }
+
+        } else {
+            // default allocator for non linux or windows system use estimate
+            pub unsafe extern "C" fn malloc_usable_size(_ptr: *const c_void) -> usize {
+                unreachable!("estimate heapsize or feature allocator needed")
+            }
+
+        }
+
+    }
+
+    /// No enclosing function defined.
+    #[inline]
+    pub fn new_enclosing_size_fn() -> Option<VoidPtrToSizeFn> {
+        None
+    }
+}
+
+/// Get a new instance of a MallocSizeOfOps
+pub fn new_malloc_size_ops() -> MallocSizeOfOps {
+    MallocSizeOfOps::new(
+        usable_size::malloc_usable_size,
+        usable_size::new_enclosing_size_fn(),
+        None,
+    )
+}
+
+/// Extension methods for `MallocSizeOf` trait, do not implement
+/// directly.
+/// It allows getting heapsize without exposing `MallocSizeOfOps`
+/// (a single default `MallocSizeOfOps` is used for each call).
+pub trait MallocSizeOfExt: MallocSizeOf {
+    /// Method to launch a heapsize measurement with a
+    /// fresh state.
+    fn malloc_size_of(&self) -> usize {
+        let mut ops = new_malloc_size_ops();
+        <Self as MallocSizeOf>::size_of(self, &mut ops)
+    }
+}
+
+impl<T: MallocSizeOf> MallocSizeOfExt for T {}
+
+#[cfg(feature = "std")]
+impl<T: MallocSizeOf> MallocSizeOf for std::sync::Arc<T> {
+    fn size_of(&self, ops: &mut MallocSizeOfOps) -> usize {
+        self.unconditional_size_of(ops)
+    }
+}

crates/parity-util-mem/src/external_impls.rs

@@ -0,0 +1,36 @@
+// Copyright (c) 2022, Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::{MallocShallowSizeOf, MallocSizeOf};
+
+// fastcrypto
+malloc_size_of_is_0!(fastcrypto::ed25519::Ed25519PublicKey);
+malloc_size_of_is_0!(fastcrypto::ed25519::Ed25519Signature);
+
+// hash_map
+malloc_size_of_is_0!(std::collections::hash_map::RandomState);
+
+// indexmap
+impl<K: MallocSizeOf, V: MallocSizeOf, S> MallocShallowSizeOf for indexmap::IndexMap<K, V, S> {
+    fn shallow_size_of(&self, _ops: &mut crate::MallocSizeOfOps) -> usize {
+        self.capacity()
+            * (std::mem::size_of::<K>()
+                + std::mem::size_of::<V>()
+                + (2 * std::mem::size_of::<usize>()))
+    }
+}
+impl<K: MallocSizeOf, V: MallocSizeOf, S> MallocSizeOf for indexmap::IndexMap<K, V, S> {
+    // This only produces a rough estimate of IndexMap size, because we cannot access private
+    // fields to measure them precisely.
+    fn size_of(&self, ops: &mut crate::MallocSizeOfOps) -> usize {
+        let mut n = self.shallow_size_of(ops);
+        if let (Some(k), Some(v)) = (K::constant_size(), V::constant_size()) {
+            n += self.len() * (k + v)
+        } else {
+            n += self
+                .iter()
+                .fold(n, |acc, (k, v)| acc + k.size_of(ops) + v.size_of(ops))
+        }
+        n
+    }
+}