More cleanups, experiments and tests.

Author: dweiss

pom.xml

@@ -50,6 +50,13 @@
     	<type>jar</type>
     	<scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>com.carrotsearch</groupId>
+      <artifactId>junit-benchmarks</artifactId>
+      <version>0.3.0</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 

src/main/java/com/carrotsearch/sizeof/BlackMagic.java

@@ -0,0 +1,92 @@
+package com.carrotsearch.sizeof;
+
+import java.lang.reflect.Field;
+import java.nio.ByteOrder;
+import java.util.Locale;
+import java.util.Map;
+import java.util.TreeMap;
+
+import sun.misc.Unsafe;
+
+/**
+ * This class contains black magic stuff based mostly on proprietary
+ * APIs and hacks. Use at your own risk.
+ */
+@SuppressWarnings({"restriction"})
+public final class BlackMagic {
+  /**
+   * Returns Unsafe if available or throw a RuntimeException.
+   */
+  public static sun.misc.Unsafe getUnsafe() {
+    try {
+      final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
+      final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+      unsafeField.setAccessible(true);
+      return (sun.misc.Unsafe) unsafeField.get(null);
+    } catch (Throwable t) {
+      throw new RuntimeException("Unsafe not available.", t);
+    }
+  }
+
+  /**
+   * Attempts to dump physical object's memory as a string.
+   */
+  public static String objectMemoryAsString(Object o) {
+    final Unsafe unsafe = getUnsafe();
+    final ByteOrder byteOrder = ByteOrder.nativeOrder();
+    
+    StringBuilder b = new StringBuilder();
+    final int obSize = (int) RamUsageEstimator.shallowSizeOf(o); 
+    for (int i = 0; i < obSize; i += 2) {
+      if ((i & 0xf) == 0) {
+        if (i > 0) b.append("\n");
+        b.append(String.format(Locale.ENGLISH, "%#06x", i));
+      }
+  
+      // we go short by short because J9 fails on odd addresses.
+      int shortValue = unsafe.getShort(o, (long) i);
+  
+      if (byteOrder == ByteOrder.BIG_ENDIAN) {
+        b.append(String.format(Locale.ENGLISH, " %02x", (shortValue >>> 8) & 0xff));
+        b.append(String.format(Locale.ENGLISH, " %02x", (shortValue & 0xff)));
+      } else {
+        b.append(String.format(Locale.ENGLISH, " %02x", (shortValue & 0xff)));
+        b.append(String.format(Locale.ENGLISH, " %02x", (shortValue >>> 8) & 0xff));
+      }
+    }
+    return b.toString();
+  }
+
+  /**
+   * Attempts to dump a layout of a class's fields in memory 
+   * (offsets from base object pointer).
+   */
+  @SuppressWarnings({"unchecked"})
+  public static String fieldsLayoutAsString(Class<?> clazz) {
+    Unsafe unsafe = getUnsafe();
+    TreeMap<Long, String> fields = new TreeMap<Long, String>(); 
+    for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
+      for (Field f : c.getDeclaredFields()) {
+        fields.put(
+            unsafe.objectFieldOffset(f),
+            f.getDeclaringClass().getSimpleName() + "." + f.getName());
+      }
+    }
+    fields.put(
+        RamUsageEstimator.shallowSizeOfInstance(clazz), "#shallowSizeOfInstance(" + clazz.getName() + ")");
+
+    StringBuilder b = new StringBuilder();
+    Object [] entries = fields.entrySet().toArray();
+    for (int i = 0; i < entries.length; i++) {
+      Map.Entry<Long, String> e    = (Map.Entry<Long, String>) entries[i];
+      Map.Entry<Long, String> next = (i + 1 < entries.length ? (Map.Entry<Long, String>) entries[i + 1] : null);
+  
+      b.append(String.format(Locale.ENGLISH,
+          "@%02d %2s %s\n", 
+          e.getKey(),
+          next == null ? "" : next.getKey() - e.getKey(),
+          e.getValue()));
+    }
+    return b.toString();
+  }
+}

src/main/java/com/carrotsearch/sizeof/RamUsageEstimator.java

@@ -33,7 +33,6 @@
  * @see #sizeOf(Object)
  */
 public final class RamUsageEstimator {
- 
   /**
    * JVM diagnostic features.
    */
@@ -232,6 +231,19 @@ private RamUsageEstimator() {}
         Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]";
   }
 
+  /**
+   * Cached information about a given class.   
+   */
+  private static final class ClassCache {
+    public final long shallowInstanceSize;
+    public final Field[] referenceFields;
+    
+    public ClassCache(long shallowInstanceSize, Field[] referenceFields) {
+      this.shallowInstanceSize = shallowInstanceSize;
+      this.referenceFields = referenceFields;
+    }    
+  }
+
   // Object with just one field to determine the object header size by getting the offset of the dummy field:
   @SuppressWarnings("unused")
   private static final class DummyOneFieldObject {
@@ -315,8 +327,9 @@ public static long sizeOf(double[] arr) {
    */
   public static long sizeOf(Object obj) {
     final Set<Object> seen = Collections.newSetFromMap(new IdentityHashMap<Object,Boolean>(64));
+    final Map<Class<?>, ClassCache> classCache = new HashMap<Class<?>, ClassCache>();
     try {
-      return measureObjectSize(obj, seen);
+      return measureObjectSize(obj, seen, classCache);
     } finally {
       // Help the GC.
       seen.clear();
@@ -334,7 +347,7 @@ public static long shallowSizeOf(Object obj) {
     if (obj == null) return 0;
     final Class<?> clz = obj.getClass();
     if (clz.isArray()) {
-      return measureArraySize(obj, null);
+      return measureArraySize(obj, null, null);
     } else {
       return shallowSizeOfInstance(clz);
     }
@@ -371,48 +384,66 @@ public static long shallowSizeOfInstance(Class<?> clazz) {
   /**
    * Recursive descend into an object.
    */
-  private static long measureObjectSize(Object obj, Set<Object> seen) {
-    if (obj == null) {
+  private static long measureObjectSize(Object obj, Set<Object> seen, Map<Class<?>, ClassCache> classCache) {
+    if (obj == null || seen.contains(obj)) {
       return 0;
     }
 
-    // skip if we have seen before
-    if (seen.contains(obj)) {
-      return 0;
-    }
-
-    // add to seen
+    // Mark as seen.
     seen.add(obj);
 
-    Class<?> clazz = obj.getClass();
+    final Class<?> clazz = obj.getClass();
     if (clazz.isArray()) {
-      return measureArraySize(obj, seen);
+      return measureArraySize(obj, seen, classCache);
     }
 
-    long objShallowSize = NUM_BYTES_OBJECT_HEADER;
-    long referencedSize = 0L;
+    // Check the cache first, otherwise walk and populate.
+    try {
+      ClassCache cachedInfo = classCache.get(clazz);
+      if (cachedInfo == null) {
+        classCache.put(clazz, cachedInfo = createCacheEntry(clazz));
+      }
+
+      long referencedSize = 0L;
+      for (Field f : cachedInfo.referenceFields) {
+        final Object o = f.get(obj);
+        if (o != null) {
+          referencedSize += measureObjectSize(o, seen, classCache);
+        }
+      }
+      return alignObjectSize(cachedInfo.shallowInstanceSize) + referencedSize;
+    } catch (IllegalAccessException e) {
+      // this should never happen as we enabled setAccessible().
+      throw new RuntimeException("Reflective field access failed?", e);
+    }
+  }
 
-    // walk type hierarchy
-    for (;clazz != null; clazz = clazz.getSuperclass()) {
-      final Field[] fields = clazz.getDeclaredFields();
+  /**
+   * Create a cached information about shallow size and reference fields for 
+   * a given class.
+   */
+  private static ClassCache createCacheEntry(final Class<?> clazz) {
+    ClassCache cachedInfo;
+    long shallowInstanceSize = NUM_BYTES_OBJECT_HEADER;
+    final ArrayList<Field> referenceFields = new ArrayList<Field>(32);
+    for (Class<?> c = clazz; c != null; c = c.getSuperclass()) {
+      final Field[] fields = c.getDeclaredFields();
       for (final Field f : fields) {
         if (!Modifier.isStatic(f.getModifiers())) {
-          objShallowSize = adjustForField(objShallowSize, f);
+          shallowInstanceSize = adjustForField(shallowInstanceSize, f);
 
           if (!f.getType().isPrimitive()) {
-            try {
-              f.setAccessible(true);
-              referencedSize += measureObjectSize(f.get(obj), seen);
-            } catch (IllegalAccessException ex) {
-              // this should never happen as we enabled setAccessible().
-              throw new RuntimeException("Cannot reflect instance field: " +
-                f.getDeclaringClass().getName() + "#" + f.getName(), ex);
-            }
+            f.setAccessible(true);
+            referenceFields.add(f);
           }
         }
       }
     }
-    return alignObjectSize(objShallowSize) + referencedSize;
+
+    cachedInfo = new ClassCache(
+        shallowInstanceSize, 
+        referenceFields.toArray(new Field[referenceFields.size()]));
+    return cachedInfo;
   }
 
   /**
@@ -445,7 +476,7 @@ private static long adjustForField(long sizeSoFar, final Field f) {
           f.getDeclaringClass().getName() + "#" + f.getName(), cause);
       }
     } else {
-      // TODO: No alignments/ subclass field alignments whatsoever? 
+      // TODO: No alignments based on field type/ subclass fields alignments?
       return sizeSoFar + fsize;
     }
   }
@@ -456,7 +487,7 @@ private static long adjustForField(long sizeSoFar, final Field f) {
    * @param seen A set of already seen objects. If <code>null</code> no references
    *             are followed and this method returns the equivalent of "shallow" array size.
    */
-  private static long measureArraySize(Object array, Set<Object> seen) {
+  private static long measureArraySize(Object array, Set<Object> seen, Map<Class<?>, ClassCache> classCache) {
     long size = NUM_BYTES_ARRAY_HEADER;
     final int len = Array.getLength(array);
     if (len > 0) {
@@ -467,7 +498,7 @@ private static long measureArraySize(Object array, Set<Object> seen) {
         size += (long) NUM_BYTES_OBJECT_REF * len;
         if (seen != null) {
           for (int i = 0; i < len; i++) {
-            size += measureObjectSize(Array.get(array, i), seen);
+            size += measureObjectSize(Array.get(array, i), seen, classCache);
           }
         }
       }

src/test/java/com/carrotsearch/sizeof/experiments/ExpFieldAlignment.java renamed to src/test/java/com/carrotsearch/sizeof/TestFieldReorderingInClassHierarchy.java

@@ -1,4 +1,4 @@
-package com.carrotsearch.sizeof.experiments;
+package com.carrotsearch.sizeof;
 
 import java.lang.reflect.Field;
 import java.util.Map;
@@ -8,16 +8,17 @@
 
 import sun.misc.Unsafe;
 
+import com.carrotsearch.sizeof.BlackMagic;
 import com.carrotsearch.sizeof.RamUsageEstimator;
+import com.carrotsearch.sizeof.experiments.WildClasses;
 
 @SuppressWarnings("restriction")
-public class ExpFieldAlignment {
+public class TestFieldReorderingInClassHierarchy {
 
   @SuppressWarnings({"deprecation"})
   @Test
   public void testFieldsOrdered() throws Exception {
-    System.out.println(RamUsageEstimator.JVM_INFO_STRING);
-    Unsafe unsafe = getUnsafe();
+    Unsafe unsafe = BlackMagic.getUnsafe();
     for (Class<?> clz : WildClasses.ALL) {
       TreeMap<Integer, Field> fields = new TreeMap<Integer, Field>(); 
       for (Class<?> c = clz; c != null; c = c.getSuperclass()) {
@@ -42,20 +43,12 @@ public void testFieldsOrdered() throws Exception {
       }
 
       if (reordered > 0) {
-        System.out.println("# packed fields: " + reordered);
-        System.out.println(ExpSubclassAlignment.dumpFields(clz));
+        System.out.println("This JVM has reordered fields!");
+        System.out.println(RamUsageEstimator.JVM_INFO_STRING);
+        System.out.println("Example class with reordered fields:");
+        System.out.println(BlackMagic.fieldsLayoutAsString(clz));
+        return;
       }
     }
   }
-
-  public static sun.misc.Unsafe getUnsafe() {
-    try {
-      final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
-      final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
-      unsafeField.setAccessible(true);
-      return (sun.misc.Unsafe) unsafeField.get(null);
-    } catch (Throwable t) {
-      throw new RuntimeException("Unsafe not available.", t);
-    }
-  }
 }

src/test/java/com/carrotsearch/sizeof/TestProcessingLargeGraph.java

@@ -0,0 +1,38 @@
+package com.carrotsearch.sizeof;
+
+import java.util.HashSet;
+import java.util.Random;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.carrotsearch.junitbenchmarks.AbstractBenchmark;
+import com.carrotsearch.junitbenchmarks.BenchmarkOptions;
+import com.carrotsearch.sizeof.experiments.WildClasses;
+
+/**
+ * Check large graph of instances with lots of classes. 
+ */
+public class TestProcessingLargeGraph extends AbstractBenchmark {
+  private static HashSet<Object> randomObjects;
+
+  @BeforeClass
+  public static void prepare() throws Exception {
+    final Random rnd = new Random(0xdeadbeef);
+    HashSet<Object> all = new HashSet<Object>();
+    Class<?>[] classes = WildClasses.ALL;
+    for (int i = 0; i < 50000; i++) {
+      all.add(classes[rnd.nextInt(classes.length)].newInstance());
+    }
+    randomObjects = all;
+  }
+
+  volatile long guard;
+
+  @BenchmarkOptions(callgc = false, benchmarkRounds = 5, warmupRounds = 3)
+  @Test
+  public void testWildClasses() {
+    guard = RamUsageEstimator.sizeOf(randomObjects);
+    System.out.println(guard);
+  }
+}