From a078fa97f9dc395115cf34bd49d80ac27e3c56d2 Mon Sep 17 00:00:00 2001 From: David Collins Date: Mon, 8 Feb 2021 09:39:18 -0700 Subject: [PATCH] Rearranged imports. Switch (non-internal) imports from "from ... import ..." for everything except for types and annotations. --- logger/classes.py | 11 ++++--- logger/logger.py | 68 +++++++++++++++++++++----------------------- logger/util.py | 43 +++++++++++++++------------- tests/test_logger.py | 59 ++++++++++++++++++-------------------- 4 files changed, 89 insertions(+), 92 deletions(-) diff --git a/logger/classes.py b/logger/classes.py index 68fcd57..ba759fd 100644 --- a/logger/classes.py +++ b/logger/classes.py @@ -2,11 +2,10 @@ from .util import runCommandWithConsole, checkIfProgramExistsInPath from abc import abstractmethod from collections import namedtuple -from inspect import stack +import inspect from multiprocessing import Process, Manager from pathlib import Path -from subprocess import run -from time import sleep +import time def traceCollector(command, **kwargs): traceName = kwargs["trace"] @@ -78,7 +77,7 @@ def start(self): def loop(self): while True: self.collect() - sleep(self.interval) + time.sleep(self.interval) @abstractmethod def collect(self): raise AbstractMethod() @@ -96,7 +95,7 @@ def __init__(self, file): class AbstractMethod(NotImplementedError): def __init__(self): - className = stack()[1].frame.f_locals['self'].__class__.__name__ - methodName = stack()[1].function + className = inspect.stack()[1].frame.f_locals['self'].__class__.__name__ + methodName = inspect.stack()[1].function super().__init__(f"{className} must implement {methodName}()") diff --git a/logger/logger.py b/logger/logger.py index 9501c43..73c66f2 100755 --- a/logger/logger.py +++ b/logger/logger.py @@ -1,18 +1,24 @@ #!/usr/bin/env python3 +from .classes import Trace, StatsCollector, Stat, traceCollector, statsCollectors +from .util import makeSVGLineChart, runCommandWithConsole, nestedSimpleNamespaceToDict from collections.abc import Iterable, Mapping import datetime -from distutils.dir_util import copy_tree +import distutils.dir_util as dir_util import json import os +from pathlib import Path +import psutil import random import re import shutil import string +import subprocess import sys import tempfile from textwrap import indent -from pathlib import Path +import time +from types import SimpleNamespace class LoggerEncoder(json.JSONEncoder): @@ -47,7 +53,7 @@ def default(self, obj): return tup elif isinstance(obj, Iterable): return [ self.default(x) for x in obj ] - elif isinstance(obj, datetime): + elif isinstance(obj, datetime.datetime): time = { '__type__': 'datetime', 'value': obj.strftime('%Y-%m-%d_%H:%M:%S:%f'), @@ -97,24 +103,13 @@ def dict_to_object(self, obj): obj['done_time'], obj['duration']) return logger elif obj['__type__'] == 'datetime': - return datetime.strptime(obj['value'], obj['format']) + return datetime.datetime.strptime(obj['value'], obj['format']) elif obj['__type__'] == 'Path': return Path(obj['value']) elif obj['__type__'] == 'tuple': return tuple(obj['items']) -from .classes import Trace, StatsCollector, Stat, traceCollector, statsCollectors -from .util import makeSVGLineChart, runCommandWithConsole, nestedSimpleNamespaceToDict -from datetime import datetime -from os import getcwd, chdir, name as osname -from psutil import disk_partitions, disk_usage, cpu_percent, virtual_memory -from random import choice -from string import ascii_lowercase -from subprocess import run -from time import time -from types import SimpleNamespace - class Logger: """ This class will keep track of commands run in the shell, their durations, @@ -201,8 +196,8 @@ def __init__(self, name, log_dir=Path.cwd(), strm_dir=None, html_file=None, # ---- self.name = name self.log_book = log if log is not None else [] - self.init_time = datetime.now() if not init_time else init_time - self.done_time = datetime.now() if not done_time else done_time + self.init_time = datetime.datetime.now() if not init_time else init_time + self.done_time = datetime.datetime.now() if not done_time else done_time self.duration = duration self.indent = indent self.is_parent = True if self.indent == 0 else False @@ -255,7 +250,7 @@ def update_done_time(self): :class:`Logger` objects who might finish their commands before the parent finalizes everything. """ - self.done_time = datetime.now() + self.done_time = datetime.datetime.now() def __update_duration(self): """ @@ -274,7 +269,7 @@ def check_duration(self): Returns: str: Duration from this object's creation until now as a string. """ - now = datetime.now() + now = datetime.datetime.now() dur = now - self.init_time return self.strfdelta(dur, "{hrs}h {min}m {sec}s") @@ -294,7 +289,7 @@ def change_log_dir(self, new_log_dir): # This only gets executed once by the top-level parent Logger object. if self.log_dir.exists(): - copy_tree(self.log_dir, new_log_dir) + dir_util.copy_tree(self.log_dir, new_log_dir) shutil.rmtree(self.log_dir) # Change the strm_dir, html_file, and log_dir for every child Logger @@ -332,8 +327,9 @@ def strfdelta(self, tdelta, fmt): Format a time delta object. Parameters: - tdelta (datetime.timedelta): Time delta object. - fmt (str): Delta format string. Use like :func:`datetime.strftime`. + tdelta (datetime.datetime.timedelta): Time delta object. + fmt (str): Delta format string. Use like + :func:`datetime.datetime.strftime`. Returns: str: String with the formatted time delta. @@ -363,7 +359,7 @@ def print(self, msg, end='\n'): print(msg, end=end) log = { 'msg': msg, - 'timestamp': str(datetime.now()), + 'timestamp': str(datetime.datetime.now()), 'cmd': None } self.log_book.append(log) @@ -602,7 +598,7 @@ def log(self, msg, cmd, cwd=None, live_stdout=False, the `stdout` and `stderr` values will be ``None``. """ - start_time = datetime.now() + start_time = datetime.datetime.now() if isinstance(cmd, list): cmd_str = ' '.join(str(x) for x in cmd) @@ -645,15 +641,15 @@ def log(self, msg, cmd, cwd=None, live_stdout=False, 'stderr': None} def run(self, command, **kwargs): - oldPWD = getcwd() + oldPWD = os.getcwd() if kwargs.get("pwd"): - chdir(kwargs.get("pwd")) + os.chdir(kwargs.get("pwd")) auxInfo = auxiliaryInformation() traceOutput, stats, completedProcess = runCommand(command, **kwargs) setattr(completedProcess, "trace", traceOutput) setattr(completedProcess, "stats", stats) if kwargs.get("pwd"): - chdir(oldPWD) + os.chdir(oldPWD) return SimpleNamespace(**completedProcess.__dict__, **auxInfo.__dict__) def auxiliaryInformation(): @@ -669,8 +665,8 @@ def auxiliaryInformation(): def auxiliaryCommandOutput(**kwargs): stdout = None - if osname in kwargs: - c = run(kwargs[osname], capture_output=True, shell=True, check=True) + if os.name in kwargs: + c = subprocess.run(kwargs[os.name], capture_output=True, shell=True, check=True) stdout = c.stdout.decode() if stdout and kwargs.get("strip"): stdout = stdout.strip() @@ -733,13 +729,13 @@ class DiskStatsCollector(StatsCollector): def __init__(self, interval, manager): super().__init__(interval, manager) self.stats = manager.dict() - self.mountpoints = [ p.mountpoint for p in disk_partitions() ] + self.mountpoints = [ p.mountpoint for p in psutil.disk_partitions() ] for m in self.mountpoints: self.stats[m] = manager.list() def collect(self): - timestamp = round(time() * 1000) + timestamp = round(time.time() * 1000) for m in self.mountpoints: - self.stats[m].append((timestamp, disk_usage(m).percent)) + self.stats[m].append((timestamp, psutil.disk_usage(m).percent)) def unproxiedStats(self): def makeStat(stat): data = list(stat) @@ -754,8 +750,8 @@ def __init__(self, interval, manager): super().__init__(interval, manager) self.stats = manager.list() def collect(self): - timestamp = round(time() * 1000) - self.stats.append((timestamp, cpu_percent(interval=None))) + timestamp = round(time.time() * 1000) + self.stats.append((timestamp, psutil.cpu_percent(interval=None))) def unproxiedStats(self): data = list(self.stats) svg = makeSVGLineChart(data) @@ -768,8 +764,8 @@ def __init__(self, interval, manager): super().__init__(interval, manager) self.stats = manager.list() def collect(self): - timestamp = round(time() * 1000) - self.stats.append((timestamp, virtual_memory().percent)) + timestamp = round(time.time() * 1000) + self.stats.append((timestamp, psutil.virtual_memory().percent)) def unproxiedStats(self): data = list(self.stats) svg = makeSVGLineChart(data) diff --git a/logger/util.py b/logger/util.py index a5e0c6c..66f0647 100644 --- a/logger/util.py +++ b/logger/util.py @@ -1,31 +1,36 @@ #!/usr/bin/env python3 from collections.abc import Iterable, Mapping -from io import StringIO, TextIOBase, BufferedIOBase, RawIOBase, IOBase -from itertools import dropwhile -from os import devnull, name as osname +from io import StringIO +import itertools +import numpy as np +import matplotlib.pyplot as pyplot +import os from queue import Queue -from subprocess import run, Popen, PIPE, DEVNULL -from time import time +import subprocess +import sys +import tempfile +import time from threading import Thread from types import SimpleNamespace -np = __import__('numpy', fromlist=["arrange"]) -pyplot = __import__('matplotlib.pyplot', fromlist=["figure", "plot", "close", "yticks"]) -sys = __import__('sys', fromlist=["stdout", "stderr"]) def checkIfProgramExistsInPath(program): - if osname == "posix": - run(f"command -V {program}", shell=True, check=True) - elif osname == "nt": - run(f"where {program}", shell=True, check=True) + if os.name == "posix": + subprocess.run(f"command -V {program}", shell=True, check=True) + elif os.name == "nt": + subprocess.run(f"where {program}", shell=True, check=True) def runCommandWithConsole(command, **kwargs): with Console(**kwargs) as console: - start = round(time() * 1000) - stdin = None if not kwargs.get("devnull_stdin") else DEVNULL - popen = Popen(command, shell=True, stdin=stdin, stdout=PIPE, stderr=PIPE) + start = round(time.time() * 1000) + stdin = None if not kwargs.get("devnull_stdin") else subprocess.DEVNULL + popen = subprocess.Popen(command, + shell=True, + stdin=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) console.attach(popen.stdout, popen.stderr) popen.wait() - finish = round(time() * 1000) + finish = round(time.time() * 1000) return SimpleNamespace( returncode = popen.returncode, args = popen.args, @@ -46,7 +51,7 @@ def makeSVGLineChart(data): pyplot.close(fig) stringIO.seek(0) lines = stringIO.readlines() - svg = "".join(dropwhile(lambda line: "= 1000 assert result.wall < 2000 @@ -305,58 +302,58 @@ def test_auxiliaryData(): result = logger.run(":") assert "PATH=" in result.environment assert logger.run("whoami").stdout.strip() == result.user - if osname == "posix": + if os.name == "posix": assert len(result.umask) == 3 or len(result.umask) == 4 assert logger.run("id -gn").stdout.strip() == result.group assert logger.run("printenv SHELL").stdout.strip() == result.shell assert logger.run("ulimit -a").stdout == result.ulimit else: - print(f"Warning: os.name is not 'posix': {osname}; umask, " + print(f"Warning: os.name is not 'posix': {os.name}; umask, " "group, shell, and ulimit not tested.") def test_workingDirectory(): logger = Logger(stack()[0][3], Path.cwd()) command = "pwd" directory = "/tmp" - if osname == "posix": + if os.name == "posix": command = "pwd" directory = "/tmp" - elif osname == "nt": + elif os.name == "nt": command = "cd" directory = "C:\\Users" else: - print(f"Warning: os.name is unrecognized: {osname}; test may fail.") + print(f"Warning: os.name is unrecognized: {os.name}; test may fail.") result = logger.run(command, pwd=directory) assert result.stdout.strip() == directory assert result.pwd == directory def test_trace(): logger = Logger(stack()[0][3], Path.cwd()) - if uname().sysname == "Linux": + if os.uname().sysname == "Linux": result = logger.run("echo letter", trace="ltrace") assert 'getenv("POSIXLY_CORRECT")' in result.trace echoLocation = logger.run("which echo").stdout.strip() result = logger.run("echo hello", trace="strace") assert f'execve("{echoLocation}' in result.trace else: - print(f"Warning: uname is not 'Linux': {uname()}; strace/ltrace " + print(f"Warning: uname is not 'Linux': {os.uname()}; strace/ltrace " "not tested.") def test_traceExpression(): logger = Logger(stack()[0][3], Path.cwd()) - if uname().sysname == "Linux": + if os.uname().sysname == "Linux": result = logger.run("echo hello", trace="ltrace", expression='getenv') assert 'getenv("POSIXLY_CORRECT")' in result.trace assert result.trace.count('\n') == 2 else: - print(f"Warning: uname is not 'Linux': {uname()}; ltrace " + print(f"Warning: uname is not 'Linux': {os.uname()}; ltrace " "expression not tested.") def test_traceSummary(): logger = Logger(stack()[0][3], Path.cwd()) - if uname().sysname == "Linux": + if os.uname().sysname == "Linux": result = logger.run("echo hello", trace="ltrace", summary=True) assert 'getenv("POSIXLY_CORRECT")' not in result.trace assert "getenv" in result.trace @@ -365,12 +362,12 @@ def test_traceSummary(): assert f'execve("{echoLocation}' not in result.trace assert "execve" in result.trace else: - print(f"Warning: uname is not 'Linux': {uname()}; strace/ltrace " + print(f"Warning: uname is not 'Linux': {os.uname()}; strace/ltrace " "summary not tested.") def test_traceExpressionAndSummary(): logger = Logger(stack()[0][3], Path.cwd()) - if uname().sysname == "Linux": + if os.uname().sysname == "Linux": echoLocation = logger.run("which echo").stdout.strip() result = logger.run("echo hello", trace="strace", @@ -387,7 +384,7 @@ def test_traceExpressionAndSummary(): assert "getenv" in result.trace assert "strcmp" not in result.trace else: - print(f"Warning: uname is not 'Linux': {uname()}; strace/ltrace " + print(f"Warning: uname is not 'Linux': {os.uname()}; strace/ltrace " "expression+summary not tested.") def test_stats(): @@ -397,15 +394,15 @@ def test_stats(): assert len(result.stats["memory"].data) < 30 assert len(result.stats["cpu"].data) > 8 assert len(result.stats["cpu"].data) < 30 - if osname == "posix": + if os.name == "posix": assert len(result.stats["disk"]["/"].data) > 8 assert len(result.stats["disk"]["/"].data) < 30 else: - print(f"Warning: os.name is not 'posix': {osname}; disk usage not fully tested.") + print(f"Warning: os.name is not 'posix': {os.name}; disk usage not fully tested.") def test_traceAndStats(): logger = Logger(stack()[0][3], Path.cwd()) - if uname().sysname == "Linux": + if os.uname().sysname == "Linux": result = logger.run("sleep 1", measure=["cpu", "memory", "disk"], interval=0.1, @@ -421,7 +418,7 @@ def test_traceAndStats(): assert len(result.stats["disk"]["/"].data) > 8 assert len(result.stats["disk"]["/"].data) < 30 else: - print(f"Warning: uname is not 'Linux': {uname()}; ltrace not tested.") + print(f"Warning: uname is not 'Linux': {os.uname()}; ltrace not tested.") def test_svg(): logger = Logger(stack()[0][3], Path.cwd()) @@ -430,7 +427,7 @@ def test_svg(): assert "" in result.stats["cpu"].svg def test_log_book_traceAndStats(): - if uname().sysname == "Linux": + if os.uname().sysname == "Linux": logger = Logger(stack()[0][3], Path.cwd()) result = logger.log("Sleep", "sleep 1", @@ -448,7 +445,7 @@ def test_log_book_traceAndStats(): assert len(logger.log_book[0]["stats"]["disk"]["/"]["data"]) > 8 assert len(logger.log_book[0]["stats"]["disk"]["/"]["data"]) < 30 else: - print(f"Warning: uname is not 'Linux': {uname()}; ltrace not tested.") + print(f"Warning: uname is not 'Linux': {os.uname()}; ltrace not tested.") def test_log_book_svg(): logger = Logger(stack()[0][3], Path.cwd())