procfs.py

#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# A Python library to read and store procfs (/proc) information on Linux.
#
# Each information storage class in this file stores original data as original
# as reasonablly possible. Translation is done when requested. It is to make it
# always possible to probe the original data.

from __future__ import print_function

import collections
import logging
import os
import re
import struct
import sys


class _NullHandler(logging.Handler):
  def emit(self, record):
    pass


_LOGGER = logging.getLogger('procfs')
_LOGGER.addHandler(_NullHandler())


class ProcStat(object):
  """Reads and stores information in /proc/pid/stat."""
  _PATTERN = re.compile(r'^'
                        '(?P<PID>-?[0-9]+) '
                        '\((?P<COMM>.+)\) '
                        '(?P<STATE>[RSDZTW]) '
                        '(?P<PPID>-?[0-9]+) '
                        '(?P<PGRP>-?[0-9]+) '
                        '(?P<SESSION>-?[0-9]+) '
                        '(?P<TTY_NR>-?[0-9]+) '
                        '(?P<TPGID>-?[0-9]+) '
                        '(?P<FLAGS>[0-9]+) '
                        '(?P<MINFIT>[0-9]+) '
                        '(?P<CMINFIT>[0-9]+) '
                        '(?P<MAJFIT>[0-9]+) '
                        '(?P<CMAJFIT>[0-9]+) '
                        '(?P<UTIME>[0-9]+) '
                        '(?P<STIME>[0-9]+) '
                        '(?P<CUTIME>[0-9]+) '
                        '(?P<CSTIME>[0-9]+) '
                        '(?P<PRIORITY>[0-9]+) '
                        '(?P<NICE>[0-9]+) '
                        '(?P<NUM_THREADS>[0-9]+) '
                        '(?P<ITREALVALUE>[0-9]+) '
                        '(?P<STARTTIME>[0-9]+) '
                        '(?P<VSIZE>[0-9]+) '
                        '(?P<RSS>[0-9]+) '
                        '(?P<RSSLIM>[0-9]+) '
                        '(?P<STARTCODE>[0-9]+) '
                        '(?P<ENDCODE>[0-9]+) '
                        '(?P<STARTSTACK>[0-9]+) '
                        '(?P<KSTKESP>[0-9]+) '
                        '(?P<KSTKEIP>[0-9]+) '
                        '(?P<SIGNAL>[0-9]+) '
                        '(?P<BLOCKED>[0-9]+) '
                        '(?P<SIGIGNORE>[0-9]+) '
                        '(?P<SIGCATCH>[0-9]+) '
                        '(?P<WCHAN>[0-9]+) '
                        '(?P<NSWAP>[0-9]+) '
                        '(?P<CNSWAP>[0-9]+) '
                        '(?P<EXIT_SIGNAL>[0-9]+) '
                        '(?P<PROCESSOR>[0-9]+) '
                        '(?P<RT_PRIORITY>[0-9]+) '
                        '(?P<POLICY>[0-9]+) '
                        '(?P<DELAYACCT_BLKIO_TICKS>[0-9]+) '
                        '(?P<GUEST_TIME>[0-9]+) '
                        '(?P<CGUEST_TIME>[0-9]+)', re.IGNORECASE)

  def __init__(self, raw, pid, vsize, rss):
    self._raw = raw
    self._pid = pid
    self._vsize = vsize
    self._rss = rss

  @staticmethod
  def load_file(stat_f):
    raw = stat_f.readlines()
    stat = ProcStat._PATTERN.match(raw[0])
    return ProcStat(raw,
                    stat.groupdict().get('PID'),
                    stat.groupdict().get('VSIZE'),
                    stat.groupdict().get('RSS'))

  @staticmethod
  def load(pid):
    try:
      with open(os.path.join('/proc', str(pid), 'stat'), 'r') as stat_f:
        return ProcStat.load_file(stat_f)
    except IOError:
      return None

  @property
  def raw(self):
    return self._raw

  @property
  def pid(self):
    return int(self._pid)

  @property
  def vsize(self):
    return int(self._vsize)

  @property
  def rss(self):
    return int(self._rss)


class ProcStatm(object):
  """Reads and stores information in /proc/pid/statm."""
  _PATTERN = re.compile(r'^'
                        '(?P<SIZE>[0-9]+) '
                        '(?P<RESIDENT>[0-9]+) '
                        '(?P<SHARE>[0-9]+) '
                        '(?P<TEXT>[0-9]+) '
                        '(?P<LIB>[0-9]+) '
                        '(?P<DATA>[0-9]+) '
                        '(?P<DT>[0-9]+)', re.IGNORECASE)

  def __init__(self, raw, size, resident, share, text, lib, data, dt):
    self._raw = raw
    self._size = size
    self._resident = resident
    self._share = share
    self._text = text
    self._lib = lib
    self._data = data
    self._dt = dt

  @staticmethod
  def load_file(statm_f):
    try:
      raw = statm_f.readlines()
    except (IOError, OSError):
      return None
    statm = ProcStatm._PATTERN.match(raw[0])
    return ProcStatm(raw,
                     statm.groupdict().get('SIZE'),
                     statm.groupdict().get('RESIDENT'),
                     statm.groupdict().get('SHARE'),
                     statm.groupdict().get('TEXT'),
                     statm.groupdict().get('LIB'),
                     statm.groupdict().get('DATA'),
                     statm.groupdict().get('DT'))

  @staticmethod
  def load(pid):
    try:
      with open(os.path.join('/proc', str(pid), 'statm'), 'r') as statm_f:
        return ProcStatm.load_file(statm_f)
    except (IOError, OSError):
      return None

  @property
  def raw(self):
    return self._raw

  @property
  def size(self):
    return int(self._size)

  @property
  def resident(self):
    return int(self._resident)

  @property
  def share(self):
    return int(self._share)

  @property
  def text(self):
    return int(self._text)

  @property
  def lib(self):
    return int(self._lib)

  @property
  def data(self):
    return int(self._data)

  @property
  def dt(self):
    return int(self._dt)


class ProcStatus(object):
  """Reads and stores information in /proc/pid/status."""
  _PATTERN = re.compile(r'^(?P<NAME>[A-Za-z0-9_]+):\s+(?P<VALUE>.*)')

  def __init__(self, raw, dct):
    self._raw = raw
    self._pid = dct.get('Pid')
    self._name = dct.get('Name')
    self._vm_peak = dct.get('VmPeak')
    self._vm_size = dct.get('VmSize')
    self._vm_lck = dct.get('VmLck')
    self._vm_pin = dct.get('VmPin')
    self._vm_hwm = dct.get('VmHWM')
    self._vm_rss = dct.get('VmRSS')
    self._vm_data = dct.get('VmData')
    self._vm_stack = dct.get('VmStk')
    self._vm_exe = dct.get('VmExe')
    self._vm_lib = dct.get('VmLib')
    self._vm_pte = dct.get('VmPTE')
    self._vm_swap = dct.get('VmSwap')

  @staticmethod
  def load_file(status_f):
    raw = status_f.readlines()
    dct = {}
    for line in raw:
      status_match = ProcStatus._PATTERN.match(line)
      if status_match:
        match_dict = status_match.groupdict()
        dct[match_dict['NAME']] = match_dict['VALUE']
      else:
        raise SyntaxError('Unknown /proc/pid/status format.')
    return ProcStatus(raw, dct)

  @staticmethod
  def load(pid):
    with open(os.path.join('/proc', str(pid), 'status'), 'r') as status_f:
      return ProcStatus.load_file(status_f)

  @property
  def raw(self):
    return self._raw

  @property
  def pid(self):
    return int(self._pid)

  @property
  def vm_peak(self):
    """Returns a high-water (peak) virtual memory size in kilo-bytes."""
    if self._vm_peak.endswith('kB'):
      return int(self._vm_peak.split()[0])
    raise ValueError('VmPeak is not in kB.')

  @property
  def vm_size(self):
    """Returns a virtual memory size in kilo-bytes."""
    if self._vm_size.endswith('kB'):
      return int(self._vm_size.split()[0])
    raise ValueError('VmSize is not in kB.')

  @property
  def vm_hwm(self):
    """Returns a high-water (peak) resident set size (RSS) in kilo-bytes."""
    if self._vm_hwm.endswith('kB'):
      return int(self._vm_hwm.split()[0])
    raise ValueError('VmHWM is not in kB.')

  @property
  def vm_rss(self):
    """Returns a resident set size (RSS) in kilo-bytes."""
    if self._vm_rss.endswith('kB'):
      return int(self._vm_rss.split()[0])
    raise ValueError('VmRSS is not in kB.')


class ProcMapsEntry(object):
  """A class representing one line in /proc/pid/maps."""

  def __init__(
      self, begin, end, readable, writable, executable, private, offset,
      major, minor, inode, name):
    self.begin = begin
    self.end = end
    self.readable = readable
    self.writable = writable
    self.executable = executable
    self.private = private
    self.offset = offset
    self.major = major
    self.minor = minor
    self.inode = inode
    self.name = name

  def as_dict(self):
    return {
        'begin': self.begin,
        'end': self.end,
        'readable': self.readable,
        'writable': self.writable,
        'executable': self.executable,
        'private': self.private,
        'offset': self.offset,
        'major': self.major,
        'minor': self.minor,
        'inode': self.inode,
        'name': self.name,
    }


class ProcMaps(object):
  """Reads and stores information in /proc/pid/maps."""

  MAPS_PATTERN = re.compile(
      r'^([a-f0-9]+)-([a-f0-9]+)\s+(.)(.)(.)(.)\s+([a-f0-9]+)\s+(\S+):(\S+)\s+'
      r'(\d+)\s*(.*)$', re.IGNORECASE)

  EXECUTABLE_PATTERN = re.compile(
      r'\S+\.(so|dll|dylib|bundle)((\.\d+)+\w*(\.\d+){0,3})?')

  def __init__(self):
    self._sorted_indexes = []
    self._dictionary = {}
    self._sorted = True

  def iter(self, condition):
    if not self._sorted:
      self._sorted_indexes.sort()
      self._sorted = True
    for index in self._sorted_indexes:
      if not condition or condition(self._dictionary[index]):
        yield self._dictionary[index]

  def __iter__(self):
    if not self._sorted:
      self._sorted_indexes.sort()
      self._sorted = True
    for index in self._sorted_indexes:
      yield self._dictionary[index]

  @staticmethod
  def load_file(maps_f):
    table = ProcMaps()
    for line in maps_f:
      table.append_line(line)
    return table

  @staticmethod
  def load(pid):
    try:
      with open(os.path.join('/proc', str(pid), 'maps'), 'r') as maps_f:
        return ProcMaps.load_file(maps_f)
    except (IOError, OSError):
      return None

  def append_line(self, line):
    entry = self.parse_line(line)
    if entry:
      self._append_entry(entry)
    return entry

  @staticmethod
  def parse_line(line):
    matched = ProcMaps.MAPS_PATTERN.match(line)
    if matched:
      return ProcMapsEntry(  # pylint: disable=W0212
          int(matched.group(1), 16),  # begin
          int(matched.group(2), 16),  # end
          matched.group(3),           # readable
          matched.group(4),           # writable
          matched.group(5),           # executable
          matched.group(6),           # private
          int(matched.group(7), 16),  # offset
          matched.group(8),           # major
          matched.group(9),           # minor
          int(matched.group(10), 10), # inode
          matched.group(11)           # name
          )
    else:
      return None

  @staticmethod
  def constants(entry):
    return entry.writable == '-' and entry.executable == '-'

  @staticmethod
  def executable(entry):
    return entry.executable == 'x'

  @staticmethod
  def executable_and_constants(entry):
    return ((entry.writable == '-' and entry.executable == '-') or
            entry.executable == 'x')

  def _append_entry(self, entry):
    if self._sorted_indexes and self._sorted_indexes[-1] > entry.begin:
      self._sorted = False
    self._sorted_indexes.append(entry.begin)
    self._dictionary[entry.begin] = entry


class ProcSmaps(object):
  """Reads and stores information in /proc/pid/smaps."""
  _SMAPS_PATTERN = re.compile(r'^(?P<NAME>[A-Za-z0-9_]+):\s+(?P<VALUE>.*)')

  class VMA(object):
    def __init__(self):
      self._size = 0
      self._rss = 0
      self._pss = 0

    def append(self, name, value):
      dct = {
        'Size': '_size',
        'Rss': '_rss',
        'Pss': '_pss',
        'Referenced': '_referenced',
        'Private_Clean': '_private_clean',
        'Shared_Clean': '_shared_clean',
        'KernelPageSize': '_kernel_page_size',
        'MMUPageSize': '_mmu_page_size',
        }
      if name in dct:
        self.__setattr__(dct[name], value)

    @property
    def size(self):
      if self._size.endswith('kB'):
        return int(self._size.split()[0])
      return int(self._size)

    @property
    def rss(self):
      if self._rss.endswith('kB'):
        return int(self._rss.split()[0])
      return int(self._rss)

    @property
    def pss(self):
      if self._pss.endswith('kB'):
        return int(self._pss.split()[0])
      return int(self._pss)

  def __init__(self, raw, total_dct, maps, vma_internals):
    self._raw = raw
    self._size = total_dct['Size']
    self._rss = total_dct['Rss']
    self._pss = total_dct['Pss']
    self._referenced = total_dct['Referenced']
    self._shared_clean = total_dct['Shared_Clean']
    self._private_clean = total_dct['Private_Clean']
    self._kernel_page_size = total_dct['KernelPageSize']
    self._mmu_page_size = total_dct['MMUPageSize']
    self._maps = maps
    self._vma_internals = vma_internals

  @staticmethod
  def load(pid):
    with open(os.path.join('/proc', str(pid), 'smaps'), 'r') as smaps_f:
      raw = smaps_f.readlines()

    vma = None
    vma_internals = collections.OrderedDict()
    total_dct = collections.defaultdict(int)
    maps = ProcMaps()
    for line in raw:
      maps_match = ProcMaps.MAPS_PATTERN.match(line)
      if maps_match:
        vma = maps.append_line(line.strip())
        vma_internals[vma] = ProcSmaps.VMA()
      else:
        smaps_match = ProcSmaps._SMAPS_PATTERN.match(line)
        if smaps_match:
          match_dict = smaps_match.groupdict()
          vma_internals[vma].append(match_dict['NAME'], match_dict['VALUE'])
          total_dct[match_dict['NAME']] += int(match_dict['VALUE'].split()[0])

    return ProcSmaps(raw, total_dct, maps, vma_internals)

  @property
  def size(self):
    return self._size

  @property
  def rss(self):
    return self._rss

  @property
  def referenced(self):
    return self._referenced

  @property
  def pss(self):
    return self._pss

  @property
  def private_clean(self):
    return self._private_clean

  @property
  def shared_clean(self):
    return self._shared_clean

  @property
  def kernel_page_size(self):
    return self._kernel_page_size

  @property
  def mmu_page_size(self):
    return self._mmu_page_size

  @property
  def vma_internals(self):
    return self._vma_internals


class ProcPagemap(object):
  """Reads and stores partial information in /proc/pid/pagemap.

  It picks up virtual addresses to read based on ProcMaps (/proc/pid/maps).
  See https://www.kernel.org/doc/Documentation/vm/pagemap.txt for details.
  """
  _BYTES_PER_PAGEMAP_VALUE = 8
  _BYTES_PER_OS_PAGE = 4096
  _VIRTUAL_TO_PAGEMAP_OFFSET = _BYTES_PER_OS_PAGE / _BYTES_PER_PAGEMAP_VALUE

  _MASK_PRESENT = 1 << 63
  _MASK_SWAPPED = 1 << 62
  _MASK_FILEPAGE_OR_SHAREDANON = 1 << 61
  _MASK_SOFTDIRTY = 1 << 55
  _MASK_PFN = (1 << 55) - 1

  class VMA(object):
    def __init__(self, vsize, present, swapped, pageframes):
      self._vsize = vsize
      self._present = present
      self._swapped = swapped
      self._pageframes = pageframes

    @property
    def vsize(self):
      return int(self._vsize)

    @property
    def present(self):
      return int(self._present)

    @property
    def swapped(self):
      return int(self._swapped)

    @property
    def pageframes(self):
      return self._pageframes

  def __init__(self, vsize, present, swapped, vma_internals, in_process_dup):
    self._vsize = vsize
    self._present = present
    self._swapped = swapped
    self._vma_internals = vma_internals
    self._in_process_dup = in_process_dup

  @staticmethod
  def load(pid, maps):
    total_present = 0
    total_swapped = 0
    total_vsize = 0
    in_process_dup = 0
    vma_internals = collections.OrderedDict()
    process_pageframe_set = set()

    try:
      pagemap_fd = os.open(
          os.path.join('/proc', str(pid), 'pagemap'), os.O_RDONLY)
    except (IOError, OSError):
      return None
    for vma in maps:
      present = 0
      swapped = 0
      vsize = 0
      pageframes = collections.defaultdict(int)
      begin_offset = ProcPagemap._offset(vma.begin)
      chunk_size = ProcPagemap._offset(vma.end) - begin_offset
      try:
        os.lseek(pagemap_fd, begin_offset, os.SEEK_SET)
        buf = os.read(pagemap_fd, chunk_size)
      except (IOError, OSError):
        return None
      if len(buf) < chunk_size:
        _LOGGER.warn('Failed to read pagemap at 0x%x in %d.' % (vma.begin, pid))
      pagemap_values = struct.unpack(
          '=%dQ' % (len(buf) / ProcPagemap._BYTES_PER_PAGEMAP_VALUE), buf)
      for pagemap_value in pagemap_values:
        vsize += ProcPagemap._BYTES_PER_OS_PAGE
        if pagemap_value & ProcPagemap._MASK_PRESENT:
          if (pagemap_value & ProcPagemap._MASK_PFN) in process_pageframe_set:
            in_process_dup += ProcPagemap._BYTES_PER_OS_PAGE
          else:
            process_pageframe_set.add(pagemap_value & ProcPagemap._MASK_PFN)
          if (pagemap_value & ProcPagemap._MASK_PFN) not in pageframes:
            present += ProcPagemap._BYTES_PER_OS_PAGE
          pageframes[pagemap_value & ProcPagemap._MASK_PFN] += 1
        if pagemap_value & ProcPagemap._MASK_SWAPPED:
          swapped += ProcPagemap._BYTES_PER_OS_PAGE
      vma_internals[vma] = ProcPagemap.VMA(vsize, present, swapped, pageframes)
      total_present += present
      total_swapped += swapped
      total_vsize += vsize
    try:
      os.close(pagemap_fd)
    except OSError:
      return None

    return ProcPagemap(total_vsize, total_present, total_swapped,
                       vma_internals, in_process_dup)

  @staticmethod
  def _offset(virtual_address):
    return virtual_address / ProcPagemap._VIRTUAL_TO_PAGEMAP_OFFSET

  @property
  def vsize(self):
    return int(self._vsize)

  @property
  def present(self):
    return int(self._present)

  @property
  def swapped(self):
    return int(self._swapped)

  @property
  def vma_internals(self):
    return self._vma_internals


class _ProcessMemory(object):
  """Aggregates process memory information from /proc for manual testing."""
  def __init__(self, pid):
    self._pid = pid
    self._maps = None
    self._pagemap = None
    self._stat = None
    self._status = None
    self._statm = None
    self._smaps = []

  def _read(self, proc_file):
    lines = []
    with open(os.path.join('/proc', str(self._pid), proc_file), 'r') as proc_f:
      lines = proc_f.readlines()
    return lines

  def read_all(self):
    self.read_stat()
    self.read_statm()
    self.read_status()
    self.read_smaps()
    self.read_maps()
    self.read_pagemap(self._maps)

  def read_maps(self):
    self._maps = ProcMaps.load(self._pid)

  def read_pagemap(self, maps):
    self._pagemap = ProcPagemap.load(self._pid, maps)

  def read_smaps(self):
    self._smaps = ProcSmaps.load(self._pid)

  def read_stat(self):
    self._stat = ProcStat.load(self._pid)

  def read_statm(self):
    self._statm = ProcStatm.load(self._pid)

  def read_status(self):
    self._status = ProcStatus.load(self._pid)

  @property
  def pid(self):
    return self._pid

  @property
  def maps(self):
    return self._maps

  @property
  def pagemap(self):
    return self._pagemap

  @property
  def smaps(self):
    return self._smaps

  @property
  def stat(self):
    return self._stat

  @property
  def statm(self):
    return self._statm

  @property
  def status(self):
    return self._status


def main(argv):
  """The main function for manual testing."""
  _LOGGER.setLevel(logging.WARNING)
  handler = logging.StreamHandler()
  handler.setLevel(logging.WARNING)
  handler.setFormatter(logging.Formatter(
      '%(asctime)s:%(name)s:%(levelname)s:%(message)s'))
  _LOGGER.addHandler(handler)

  pids = []
  for arg in argv[1:]:
    try:
      pid = int(arg)
    except ValueError:
      raise SyntaxError("%s is not an integer." % arg)
    else:
      pids.append(pid)

  procs = {}
  for pid in pids:
    procs[pid] = _ProcessMemory(pid)
    procs[pid].read_all()

    print('=== PID: %d ===' % pid)

    print('   stat: %d' % procs[pid].stat.vsize)
    print('  statm: %d' % (procs[pid].statm.size * 4096))
    print(' status: %d (Peak:%d)' % (procs[pid].status.vm_size * 1024,
                                     procs[pid].status.vm_peak * 1024))
    print('  smaps: %d' % (procs[pid].smaps.size * 1024))
    print('pagemap: %d' % procs[pid].pagemap.vsize)
    print('   stat: %d' % (procs[pid].stat.rss * 4096))
    print('  statm: %d' % (procs[pid].statm.resident * 4096))
    print(' status: %d (Peak:%d)' % (procs[pid].status.vm_rss * 1024,
                                     procs[pid].status.vm_hwm * 1024))
    print('  smaps: %d' % (procs[pid].smaps.rss * 1024))
    print('pagemap: %d' % procs[pid].pagemap.present)

  return 0


if __name__ == '__main__':
  sys.exit(main(sys.argv))