[observatory] Display process-wide memory usage with heap snapshots.

Include major memory users known to the VM: isolate heaps, profiling buffers, timeline buffers.

Change-Id: I2580ad74b5d4d07c5c75fde28bb7a5d71fddb09b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/127382
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Ben Konyi <bkonyi@google.com>

Author: rmacnak-google

runtime/observatory/lib/object_graph.dart

@@ -17,6 +17,8 @@ class _ReadStream {
 
   _ReadStream(this._buffer);
 
+  bool atEnd() => _position >= _buffer.length;
+
   int readByte() => _buffer[_position++];
 
   /// Read one ULEB128 number.
@@ -419,6 +421,8 @@ abstract class SnapshotGraph {
   Iterable<SnapshotClass> get classes;
   Iterable<SnapshotObject> get objects;
 
+  Map<String, int> get processPartitions;
+
   factory SnapshotGraph(Uint8List encoded) => new _SnapshotGraph(encoded);
   Stream<String> process();
 }
@@ -496,6 +500,8 @@ class _SnapshotGraph implements SnapshotGraph {
     return result;
   }
 
+  final processPartitions = new Map<String, int>();
+
   Stream<String> process() {
     final controller = new StreamController<String>.broadcast();
     (() async {
@@ -512,6 +518,9 @@ class _SnapshotGraph implements SnapshotGraph {
 
       controller.add("Loading external properties...");
       await new Future(() => _readExternalProperties(stream));
+
+      controller.add("Loading process partitions...");
+      await new Future(() => _readProcessPartitions(stream));
       stream = null;
 
       controller.add("Compute class table...");
@@ -730,6 +739,17 @@ class _SnapshotGraph implements SnapshotGraph {
     _externalSizes = externalSizes;
   }
 
+  void _readProcessPartitions(_ReadStream stream) {
+    // So it isn't null when loading older saved snapshots.
+    processPartitions["RSS"] = 0;
+
+    while (!stream.atEnd()) {
+      var name = stream.readUtf8();
+      var size = stream.readUnsigned();
+      processPartitions[name] = size;
+    }
+  }
+
   void _computeClassTable() {
     var N = _N;
     var classes = _classes;

runtime/observatory/lib/src/elements/heap_snapshot.dart

@@ -41,6 +41,7 @@ enum HeapSnapshotTreeMode {
   classesTreeMap,
   successors,
   predecessors,
+  process,
 }
 
 class DominatorTreeMap extends TreeMap<SnapshotObject> {
@@ -116,6 +117,30 @@ class ClassesShallowTreeMap extends TreeMap<SnapshotClass> {
   }
 }
 
+class ProcessTreeMap extends TreeMap<String> {
+  static const String root = "RSS";
+  HeapSnapshotElement element;
+  M.HeapSnapshot snapshot;
+
+  ProcessTreeMap(this.element, this.snapshot);
+
+  int getSize(String node) => snapshot.processPartitions[node];
+  String getType(String node) => node;
+  String getLabel(String node) => node;
+  String getParent(String node) => node == root ? null : root;
+  Iterable<String> getChildren(String node) {
+    if (node == root) {
+      var children = snapshot.processPartitions.keys.toList();
+      children.remove(root);
+      return children;
+    }
+    return <String>[];
+  }
+
+  void onSelect(String node) {}
+  void onDetails(String node) {}
+}
+
 // Using `null` to represent the root.
 class ClassesOwnershipTreeMap extends TreeMap<SnapshotClass> {
   HeapSnapshotElement element;
@@ -578,6 +603,27 @@ class HeapSnapshotElement extends CustomElement implements Renderable {
             ..children = [content]
         ]);
         break;
+      case HeapSnapshotTreeMode.process:
+        final content = new DivElement();
+        content.style.border = '1px solid black';
+        content.style.width = '100%';
+        content.style.height = '100%';
+        content.text = 'Performing layout...';
+        Timer.run(() {
+          // Generate the treemap after the content div has been added to the
+          // document so that we can ask the browser how much space is
+          // available for treemap layout.
+          new ProcessTreeMap(this, _snapshot)
+              .showIn(ProcessTreeMap.root, content);
+        });
+        report.addAll([
+          new DivElement()
+            ..classes = ['content-centered-big']
+            ..style.width = '100%'
+            ..style.height = '100%'
+            ..children = [content]
+        ]);
+        break;
       default:
         break;
     }
@@ -910,6 +956,8 @@ class HeapSnapshotElement extends CustomElement implements Renderable {
         return 'Successors / outgoing references';
       case HeapSnapshotTreeMode.predecessors:
         return 'Predecessors / incoming references';
+      case HeapSnapshotTreeMode.process:
+        return 'Process memory usage';
     }
     throw new Exception('Unknown HeapSnapshotTreeMode: $mode');
   }

runtime/observatory/lib/src/heap_snapshot/heap_snapshot.dart

@@ -15,6 +15,7 @@ class HeapSnapshot implements M.HeapSnapshot {
   HeapSnapshotMergedDominatorNode mergedDominatorTree;
   List<SnapshotClass> classes;
   SnapshotObject get root => graph.root;
+  Map<String, int> get processPartitions => graph.processPartitions;
   Uint8List encoded;
 
   Stream<String> loadProgress(S.Isolate isolate, Uint8List encoded) {

runtime/observatory/lib/src/models/objects/heap_snapshot.dart

@@ -10,6 +10,7 @@ abstract class HeapSnapshot {
   SnapshotObject get root;
   HeapSnapshotMergedDominatorNode get mergedDominatorTree;
   Iterable<SnapshotClass> get classes;
+  Map<String, int> get processPartitions;
   Uint8List get encoded;
 }
 

runtime/vm/object_graph.cc

@@ -11,6 +11,7 @@
 #include "vm/native_symbol.h"
 #include "vm/object.h"
 #include "vm/object_store.h"
+#include "vm/profiler.h"
 #include "vm/raw_object.h"
 #include "vm/raw_object_fields.h"
 #include "vm/reusable_handles.h"
@@ -1007,6 +1008,35 @@ void HeapSnapshotWriter::Write() {
     isolate()->VisitWeakPersistentHandles(&visitor);
   }
 
+  {
+    WriteUtf8("RSS");
+    WriteUnsigned(Service::CurrentRSS());
+
+    WriteUtf8("Dart Profiler Samples");
+    WriteUnsigned(Profiler::Size());
+
+    WriteUtf8("Dart Timeline Events");
+    WriteUnsigned(Timeline::recorder()->Size());
+
+    class WriteIsolateHeaps : public IsolateVisitor {
+     public:
+      explicit WriteIsolateHeaps(HeapSnapshotWriter* writer)
+          : writer_(writer) {}
+
+      virtual void VisitIsolate(Isolate* isolate) {
+        writer_->WriteUtf8(isolate->name());
+        writer_->WriteUnsigned((isolate->heap()->TotalCapacityInWords() +
+                                isolate->heap()->TotalExternalInWords()) *
+                               kWordSize);
+      }
+
+     private:
+      HeapSnapshotWriter* const writer_;
+    };
+    WriteIsolateHeaps visitor(this);
+    Isolate::VisitIsolates(&visitor);
+  }
+
   ClearObjectIds();
   Flush(true);
 }

runtime/vm/profiler.h

@@ -96,6 +96,7 @@ class Profiler : public AllStatic {
     // Copies the counter values.
     return counters_;
   }
+  inline static intptr_t Size();
 
  private:
   static void DumpStackTrace(uword sp, uword fp, uword pc, bool for_crash);
@@ -674,6 +675,8 @@ class SampleBuffer {
 
   ProcessedSampleBuffer* BuildProcessedSampleBuffer(SampleFilter* filter);
 
+  intptr_t Size() { return memory_->size(); }
+
  protected:
   ProcessedSample* BuildProcessedSample(Sample* sample,
                                         const CodeLookupTable& clt);
@@ -705,6 +708,17 @@ class AllocationSampleBuffer : public SampleBuffer {
   DISALLOW_COPY_AND_ASSIGN(AllocationSampleBuffer);
 };
 
+intptr_t Profiler::Size() {
+  intptr_t size = 0;
+  if (sample_buffer_ != nullptr) {
+    size += sample_buffer_->Size();
+  }
+  if (allocation_sample_buffer_ != nullptr) {
+    size += allocation_sample_buffer_->Size();
+  }
+  return size;
+}
+
 // A |ProcessedSample| is a combination of 1 (or more) |Sample|(s) that have
 // been merged into a logical sample. The raw data may have been processed to
 // improve the quality of the stack trace.

runtime/vm/timeline.cc

@@ -1161,6 +1161,10 @@ TimelineEventFixedBufferRecorder::~TimelineEventFixedBufferRecorder() {
   delete memory_;
 }
 
+intptr_t TimelineEventFixedBufferRecorder::Size() {
+  return memory_->size();
+}
+
 #ifndef PRODUCT
 void TimelineEventFixedBufferRecorder::PrintJSONEvents(
     JSONArray* events,

runtime/vm/timeline.h

@@ -719,6 +719,8 @@ class TimelineEventRecorder {
 
   void FinishBlock(TimelineEventBlock* block);
 
+  virtual intptr_t Size() = 0;
+
  protected:
 #ifndef PRODUCT
   void WriteTo(const char* directory);
@@ -771,6 +773,8 @@ class TimelineEventFixedBufferRecorder : public TimelineEventRecorder {
   void PrintTraceEvent(JSONStream* js, TimelineEventFilter* filter);
 #endif
 
+  intptr_t Size();
+
  protected:
   TimelineEvent* StartEvent();
   void CompleteEvent(TimelineEvent* event);
@@ -857,6 +861,7 @@ class TimelineEventEndlessRecorder : public TimelineEventRecorder {
 #endif
 
   const char* name() const { return ENDLESS_RECORDER_NAME; }
+  intptr_t Size() { return block_index_ * sizeof(TimelineEventBlock); }
 
  protected:
   TimelineEvent* StartEvent();
@@ -929,6 +934,7 @@ class TimelineEventFuchsiaRecorder : public TimelineEventPlatformRecorder {
   virtual ~TimelineEventFuchsiaRecorder() {}
 
   const char* name() const { return FUCHSIA_RECORDER_NAME; }
+  intptr_t Size() { return 0; }
 
  private:
   void OnEvent(TimelineEvent* event);
@@ -949,6 +955,7 @@ class TimelineEventSystraceRecorder : public TimelineEventPlatformRecorder {
                                 intptr_t buffer_size);
 
   const char* name() const { return SYSTRACE_RECORDER_NAME; }
+  intptr_t Size() { return 0; }
 
  private:
   void OnEvent(TimelineEvent* event);

runtime/vm/timeline_test.cc

@@ -275,6 +275,8 @@ class EventCounterRecorder : public TimelineEventCallbackRecorder {
 
   intptr_t CountFor(TimelineEvent::EventType type) { return counts_[type]; }
 
+  intptr_t Size() { return -1; }
+
  private:
   intptr_t counts_[TimelineEvent::kNumEventTypes];
 };