Small refactor to memory_inspector.

Some small architectural changes required for the upcoming www UI.
The main change introduced by this CL is moving the deps inclusion 
from run_tests to the main module initializer.
Rationale: in the next cl, a new binary (start_web_ui) will need to
include the same deps, so moving that to a common place.

BUG=340294
NOTRY=true

Review URL: https://codereview.chromium.org/192033003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@256204 0039d316-1c4b-4281-b951-d872f2087c98

tools/memory_inspector/memory_inspector/__init__.py

@@ -0,0 +1,31 @@
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+
+
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+
+
+def RegisterAllBackends():
+  """Registers all the known backends."""
+  from memory_inspector.core import backends
+  from memory_inspector.backends.android import android_backend
+  backends.Register(android_backend.AndroidBackend())
+
+
+def _IncludeDeps():
+  """Imports all the project dependencies."""
+  chromium_dir = os.path.abspath(os.path.join(ROOT_DIR, os.pardir, os.pardir))
+  assert(os.path.isdir(chromium_dir)), 'Cannot find chromium ' + chromium_dir
+
+  sys.path += [
+      ROOT_DIR,
+
+      # Include all dependencies.
+      os.path.join(chromium_dir, 'build', 'android'),  # For pylib.
+  ]
+
+_IncludeDeps()

tools/memory_inspector/memory_inspector/backends/android/android_backend.py

@@ -40,9 +40,10 @@ class AndroidBackend(backends.Backend):
       'adb_path': 'Path of directory containing the adb binary',
       'toolchain_path': 'Path of toolchain (for addr2line)'}
 
-  def __init__(self, settings=None):
+  def __init__(self):
     super(AndroidBackend, self).__init__(
         settings=backends.Settings(AndroidBackend._SETTINGS_KEYS))
+    self._devices = {}  # 'device id' -> |Device|.
 
   def EnumerateDevices(self):
     # If a custom adb_path has been setup through settings, prepend that to the
@@ -51,8 +52,13 @@ def EnumerateDevices(self):
         not os.environ['PATH'].startswith(self.settings['adb_path'])):
       os.environ['PATH'] = os.pathsep.join([self.settings['adb_path'],
                                            os.environ['PATH']])
-    for device in android_commands.GetAttachedDevices():
-      yield AndroidDevice(self, android_commands.AndroidCommands(device))
+    for device_id in android_commands.GetAttachedDevices():
+      device = self._devices.get(device_id)
+      if not device:
+        device = AndroidDevice(
+          self, android_commands.AndroidCommands(device_id))
+        self._devices[device_id] = device
+      yield device
 
   @property
   def name(self):
@@ -73,7 +79,10 @@ def __init__(self, backend, adb):
     self._id = adb.GetDevice()
     self._name = adb.GetProductModel()
     self._sys_stats = None
+    self._last_device_stats = None
     self._sys_stats_last_update = None
+    self._processes = {}  # pid (int) -> |Process|
+    self._initialized = False
 
   def Initialize(self):
     """Starts adb root and deploys the prebuilt binaries on initialization."""
@@ -84,6 +93,7 @@ def Initialize(self):
                                          _MEMDUMP_PATH_ON_DEVICE)
     self._DeployPrebuiltOnDeviceIfNeeded(_PSEXT_PREBUILT_PATH,
                                          _PSEXT_PATH_ON_DEVICE)
+    self._initialized = True
 
   def EnableMmapTracing(self, enabled):
     """Nothing to do here. memdump is already deployed in Initialize()."""
@@ -98,6 +108,7 @@ def IsNativeAllocTracingEnabled(self):
 
   def EnableNativeAllocTracing(self, enabled):
     """Enables libc.debug.malloc and restarts the shell."""
+    assert(self._initialized)
     prop_value = '1' if enabled else ''
     self.adb.system_properties[_DLMALLOC_DEBUG_SYSPROP] = prop_value
     assert(self.IsNativeAllocTracingEnabled())
@@ -106,24 +117,19 @@ def EnableNativeAllocTracing(self, enabled):
 
   def ListProcesses(self):
     """Returns a sequence of |AndroidProcess|."""
-    sys_stats = self.UpdateAndGetSystemStats()
-    for pid, proc in sys_stats['processes'].iteritems():
-      yield AndroidProcess(self, int(pid), proc['name'])
+    self._RefreshProcessesList()
+    return self._processes.itervalues()
 
   def GetProcess(self, pid):
     """Returns an instance of |AndroidProcess| (None if not found)."""
     assert(isinstance(pid, int))
-    sys_stats = self.UpdateAndGetSystemStats()
-    proc = sys_stats['processes'].get(str(pid))
-    if not proc:
-      return None
-    return AndroidProcess(self, pid, proc['name'])
+    self._RefreshProcessesList()
+    return self._processes.get(pid)
 
   def GetStats(self):
     """Returns an instance of |DeviceStats| with the OS CPU/Memory stats."""
-    old = self._sys_stats
     cur = self.UpdateAndGetSystemStats()
-    old = old or cur  # Handle 1st call case.
+    old = self._last_device_stats or cur  # Handle 1st call case.
     uptime = cur['time']['ticks'] / cur['time']['rate']
     ticks = max(1, cur['time']['ticks'] - old['time']['ticks'])
 
@@ -133,22 +139,32 @@ def GetStats(self):
           'usr': 100 * (cur['cpu'][i]['usr'] - old['cpu'][i]['usr']) / ticks,
           'sys': 100 * (cur['cpu'][i]['sys'] - old['cpu'][i]['sys']) / ticks,
           'idle': 100 * (cur['cpu'][i]['idle'] - old['cpu'][i]['idle']) / ticks}
-      # The idle tick count on many Linux kernels stays frozen when the CPU is
+      # The idle tick count on many Linux kernels is frozen when the CPU is
       # offline, and bumps up (compensating all the offline period) when it
-      # reactivates. For this reason it needs to be saturated at 100.
-      cpu_time['idle'] = min(cpu_time['idle'],
-                             100 - cpu_time['usr'] - cpu_time['sys'])
+      # reactivates. For this reason it needs to be saturated at [0, 100].
+      cpu_time['idle'] = max(0, min(cpu_time['idle'],
+                                    100 - cpu_time['usr'] - cpu_time['sys']))
+
       cpu_times.append(cpu_time)
+
+    memory_stats = {'Free': cur['mem']['MemFree:'],
+                    'Cache': cur['mem']['Buffers:'] + cur['mem']['Cached:'],
+                    'Swap': cur['mem']['SwapCached:'],
+                    'Anonymous': cur['mem']['AnonPages:'],
+                    'Kernel': cur['mem']['VmallocUsed:']}
+    self._last_device_stats = cur
+
     return backends.DeviceStats(uptime=uptime,
-                               cpu_times=cpu_times,
-                               memory_stats=cur['mem'])
+                                cpu_times=cpu_times,
+                                memory_stats=memory_stats)
 
   def UpdateAndGetSystemStats(self):
-    """Grabs and caches system stats through ps_ext (max cache TTL = 1s).
+    """Grabs and caches system stats through ps_ext (max cache TTL = 0.5s).
 
     Rationale of caching: avoid invoking adb too often, it is slow.
     """
-    max_ttl = datetime.timedelta(seconds=1)
+    assert(self._initialized)
+    max_ttl = datetime.timedelta(seconds=0.5)
     if (self._sys_stats_last_update and
         datetime.datetime.now() - self._sys_stats_last_update <= max_ttl):
       return self._sys_stats
@@ -161,6 +177,19 @@ def UpdateAndGetSystemStats(self):
     self._sys_stats_last_update = datetime.datetime.now()
     return self._sys_stats
 
+  def _RefreshProcessesList(self):
+    sys_stats = self.UpdateAndGetSystemStats()
+    processes_to_delete = set(self._processes.keys())
+    for pid, proc in sys_stats['processes'].iteritems():
+      pid = int(pid)
+      process = self._processes.get(pid)
+      if not process or process.name != proc['name']:
+        process = AndroidProcess(self, int(pid), proc['name'])
+        self._processes[pid] = process
+      processes_to_delete.discard(pid)
+    for pid in processes_to_delete:
+      del self._processes[pid]
+
   def _DeployPrebuiltOnDeviceIfNeeded(self, local_path, path_on_device):
     # TODO(primiano): check that the md5 binary is built-in also on pre-KK.
     # Alternatively add tools/android/md5sum to prebuilts and use that one.
@@ -236,6 +265,8 @@ def GetStats(self):
         run_time=run_time,
         cpu_usage=cpu_usage,
         vm_rss=cur_proc_stats['vm_rss'],
-        page_faults=cur_proc_stats['maj_faults'] + cur_proc_stats['min_faults'])
+        page_faults=(
+            (cur_proc_stats['maj_faults'] + cur_proc_stats['min_faults']) -
+            (old_proc_stats['maj_faults'] + old_proc_stats['min_faults'])))
     self._last_sys_stats = cur_sys_stats
     return proc_stats

tools/memory_inspector/memory_inspector/backends/android/android_backend_unittest.py

@@ -66,10 +66,10 @@
   "processes": {
     "1": {
        "name": "foo", "n_threads": 42, "start_time": 1000000, "user_time": 82,
-       "sys_time": 445, "min_faults": 918, "maj_faults": 2, "vm_rss": 528},
+       "sys_time": 445, "min_faults": 0, "maj_faults": 0, "vm_rss": 528},
     "2": {
        "name": "bar", "n_threads": 142, "start_time": 1, "user_time": 82,
-       "sys_time": 445, "min_faults": 918, "maj_faults": 0, "vm_rss": 528}}
+       "sys_time": 445, "min_faults": 0, "maj_faults": 0, "vm_rss": 528}}
 }
 """
 
@@ -125,7 +125,7 @@ def runTest(self):
     self.assertEqual(stats.cpu_usage, 0)
     self.assertEqual(stats.run_time, 1)
     self.assertEqual(stats.vm_rss, 528)
-    self.assertEqual(stats.page_faults, 920)
+    self.assertEqual(stats.page_faults, 0)
 
     # Test memdump parsing.
     mmaps = processes[0].DumpMemoryMaps()

tools/memory_inspector/memory_inspector/core/backends.py

@@ -19,14 +19,18 @@ def ListDevices():
       yield device
 
 
+def GetBackend(backend_name):
+  """Retrieves a specific backend given its name."""
+  return _backends.get(backend_name, None)
+
+
 def GetDevice(backend_name, device_id):
   """Retrieves a specific device given its backend name and device id."""
-  for backend in _backends.itervalues():
-    if backend.name != backend_name:
-      continue
-    for device in backend.EnumerateDevices():
-      if device.id != device_id:
-        continue
+  backend = GetBackend(backend_name)
+  if not backend:
+    return None
+  for device in backend.EnumerateDevices():
+    if device.id == device_id:
       return device
   return None
 

tools/memory_inspector/memory_inspector/data/serialization.py

@@ -14,6 +14,7 @@
 
 import json
 
+from memory_inspector.core import backends
 from memory_inspector.core import memory_map
 from memory_inspector.core import native_heap
 from memory_inspector.core import stacktrace
@@ -55,6 +56,9 @@ def default(self, obj):  # pylint: disable=E0202
               'exec_file_rel_path': obj.exec_file_rel_path,
               'offset': obj.offset}
 
+    if isinstance(obj, (backends.DeviceStats, backends.ProcessStats)):
+      return obj.__dict__
+
     return json.JSONEncoder.default(self, obj)
 
 

tools/memory_inspector/run_tests

@@ -4,32 +4,22 @@
 # found in the LICENSE file.
 
 import logging
-import os
+import memory_inspector
 import sys
 import unittest
 
 
 if __name__ == '__main__':
-  cur_dir = os.path.abspath(os.path.dirname(__file__))
-  chromium_dir = os.path.abspath(os.path.join(cur_dir, os.pardir, os.pardir))
-
-  sys.path += [
-      cur_dir,
-
-      # Include all dependencies.
-      os.path.join(chromium_dir, 'build', 'android'),  # For pylib.
-  ]
-
   logging.basicConfig(
       level=logging.DEBUG if '-v' in sys.argv else logging.WARNING,
       format='%(levelname)5s %(filename)15s(%(lineno)3d): %(message)s')
 
   suite = unittest.TestSuite()
   loader = unittest.TestLoader()
-  suite.addTests(loader.discover(start_dir=cur_dir, pattern='*_unittest.py'))
-
+  suite.addTests(loader.discover(start_dir=memory_inspector.ROOT_DIR,
+                                 pattern='*_unittest.py'))
   res = unittest.TextTestRunner(verbosity=2).run(suite)
   if res.wasSuccessful():
     sys.exit(0)
   else:
-    sys.exit(1)
+    sys.exit(1)