feat(core): Add tracing spans to transaction storage layer (#8796)

# Description of change

Add tracing spans to transaction storage layer.

## Links to any relevant issues

Close #8691 

## How the change has been tested

- [x] Basic tests (linting, compilation, formatting, unit/integration
tests)
- [ ] Patch-specific tests (correctness, functionality coverage)
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have checked that new and existing unit tests pass locally with
my changes

Author: jkrvivian

crates/iota-core/src/execution_cache/passthrough_cache.rs

@@ -91,6 +91,7 @@ impl PassthroughCache {
 }
 
 impl ObjectCacheRead for PassthroughCache {
+    #[instrument(level = "trace", skip_all, fields(package_id))]
     fn try_get_package_object(&self, package_id: &ObjectID) -> IotaResult<Option<PackageObject>> {
         self.package_cache
             .get_package_object(package_id, &*self.store)
@@ -101,10 +102,12 @@ impl ObjectCacheRead for PassthroughCache {
             .force_reload_system_packages(system_package_ids.iter().cloned(), self);
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id = ?id))]
     fn try_get_object(&self, id: &ObjectID) -> IotaResult<Option<Object>> {
         self.store.try_get_object(id).map_err(Into::into)
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id, version))]
     fn try_get_object_by_key(
         &self,
         object_id: &ObjectID,
@@ -113,13 +116,15 @@ impl ObjectCacheRead for PassthroughCache {
         Ok(self.store.try_get_object_by_key(object_id, version)?)
     }
 
+    #[instrument(level = "trace", skip_all)]
     fn try_multi_get_objects_by_key(
         &self,
         object_keys: &[ObjectKey],
     ) -> Result<Vec<Option<Object>>, IotaError> {
         Ok(self.store.try_multi_get_objects_by_key(object_keys)?)
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id, version))]
     fn try_object_exists_by_key(
         &self,
         object_id: &ObjectID,
@@ -128,24 +133,28 @@ impl ObjectCacheRead for PassthroughCache {
         self.store.object_exists_by_key(object_id, version)
     }
 
+    #[instrument(level = "trace", skip_all)]
     fn try_multi_object_exists_by_key(&self, object_keys: &[ObjectKey]) -> IotaResult<Vec<bool>> {
         self.store.multi_object_exists_by_key(object_keys)
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id))]
     fn try_get_latest_object_ref_or_tombstone(
         &self,
         object_id: ObjectID,
     ) -> IotaResult<Option<ObjectRef>> {
         self.store.get_latest_object_ref_or_tombstone(object_id)
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id))]
     fn try_get_latest_object_or_tombstone(
         &self,
         object_id: ObjectID,
     ) -> Result<Option<(ObjectKey, ObjectOrTombstone)>, IotaError> {
         self.store.get_latest_object_or_tombstone(object_id)
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id, version_bound))]
     fn try_find_object_lt_or_eq_version(
         &self,
         object_id: ObjectID,

crates/iota-core/src/execution_cache/writeback_cache.rs

@@ -1308,6 +1308,7 @@ impl ExecutionCacheCommit for WritebackCache {
 }
 
 impl ObjectCacheRead for WritebackCache {
+    #[instrument(level="trace", skip_all, fields(package_id=?package_id))]
     fn try_get_package_object(&self, package_id: &ObjectID) -> IotaResult<Option<PackageObject>> {
         self.metrics
             .record_cache_request("package", "package_cache");
@@ -1369,10 +1370,12 @@ impl ObjectCacheRead for WritebackCache {
 
     // get_object and variants.
 
+    #[instrument(level = "trace", skip_all, fields(object_id=?id))]
     fn try_get_object(&self, id: &ObjectID) -> IotaResult<Option<Object>> {
         self.get_object_impl("object_latest", id)
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id, version))]
     fn try_get_object_by_key(
         &self,
         object_id: &ObjectID,
@@ -1387,6 +1390,7 @@ impl ObjectCacheRead for WritebackCache {
         }
     }
 
+    #[instrument(level = "trace", skip_all)]
     fn try_multi_get_objects_by_key(
         &self,
         object_keys: &[ObjectKey],
@@ -1407,6 +1411,7 @@ impl ObjectCacheRead for WritebackCache {
         )
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id, version))]
     fn try_object_exists_by_key(
         &self,
         object_id: &ObjectID,
@@ -1421,6 +1426,7 @@ impl ObjectCacheRead for WritebackCache {
         }
     }
 
+    #[instrument(level = "trace", skip_all)]
     fn try_multi_object_exists_by_key(&self, object_keys: &[ObjectKey]) -> IotaResult<Vec<bool>> {
         try_do_fallback_lookup(
             object_keys,
@@ -1438,6 +1444,7 @@ impl ObjectCacheRead for WritebackCache {
         )
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id))]
     fn try_get_latest_object_ref_or_tombstone(
         &self,
         object_id: ObjectID,
@@ -1455,6 +1462,7 @@ impl ObjectCacheRead for WritebackCache {
         }
     }
 
+    #[instrument(level = "trace", skip_all, fields(object_id))]
     fn try_get_latest_object_or_tombstone(
         &self,
         object_id: ObjectID,
@@ -2055,6 +2063,7 @@ impl TransactionCacheRead for WritebackCache {
 }
 
 impl ExecutionCacheWrite for WritebackCache {
+    #[instrument(level = "debug", skip_all)]
     fn try_acquire_transaction_locks(
         &self,
         epoch_store: &AuthorityPerEpochStore,