Documentation

- Add inline code documentation

Author: jelu

dsc_datatool/__init__.py

@@ -1,10 +1,35 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
-
-    DSC datatool
-
-    :copyright: 2020 OARC, Inc.
+"""dsc_datatool
+
+The main Python module for the command line tool `dsc-datatool`, see
+`man dsc-datatool` on how to run it.
+
+On runtime it will load all plugins under the following module path:
+- dsc_datatool.input
+- dsc_datatool.output
+- dsc_datatool.generator
+- dsc_datatool.transformer
+
+Each plugin category should base it class on one of the follow superclasses:
+- dsc_datatool.Input
+- dsc_datatool.Output
+- dsc_datatool.Generator
+- dsc_datatool.Transformer
+
+Doing so it will be automatically registered as available and indexed in
+the following public dicts using the class name:
+- inputs
+- outputs
+- generators
+- transformers
+
+Example of an output:
+
+    from dsc_datatool import Output
+    class ExampleOutput(Output):
+        def process(self, datasets)
+            ...
+
+:copyright: 2020 OARC, Inc.
 """
 
 __version__ = '1.0.0'
@@ -27,87 +52,181 @@
 
 
 class Dataset(object):
+    """A representation of a DSC dataset
+
+    A DSC dataset is one to two dimensional structure where the last
+    dimension holds an array of values and counters.
+
+    It is based on the XML structure of DSC:
+
+        <array name="pcap_stats" dimensions="2" start_time="1563520560" stop_time="1563520620">
+          <dimension number="1" type="ifname"/>
+          <dimension number="2" type="pcap_stat"/>
+          <data>
+            <ifname val="eth0">
+              <pcap_stat val="filter_received" count="5625"/>
+              <pcap_stat val="pkts_captured" count="4894"/>
+              <pcap_stat val="kernel_dropped" count="731"/>
+            </ifname>
+          </data>
+        </array>
+
+    Attributes:
+    - name: The name of the dataset
+    - start_time: The start time of the dataset in seconds
+    - stop_time: The stop time of the dataset in seconds
+    - dimensions: An array with `Dimension`, the first dimension
+    """
     name = None
     start_time = None
     stop_time = None
     dimensions = None
 
+
     def __init__(self):
         self.dimensions = []
 
+
     def __repr__(self):
         return '<Dataset name=%r dimension=%r>' % (self.name, self.dimensions)
 
 
 class Dimension(object):
+    """A representation of a DSC dimension
+
+    A DSC dataset dimension which can be the first or second dimension,
+    see `Dataset` for more information.
+
+    Attributes:
+    - name: The name of the dimension
+    - value: Is set to the value of the dimension if it's the first dimension
+    - values: A dict of values with corresponding counters if it's the second dimension
+    """
     name = None
     value = None
     values = None
     dimensions = None
 
+
     def __init__(self, name):
         self.name = name
         self.values = {}
         self.dimensions = []
 
+
     def __repr__(self):
         return '<Dimension name=%r value=%r dimension=%r>' % (self.name, self.values or self.value, self.dimensions)
 
 
 class Input(object):
+    """Base class of an input plugin"""
+
+
     def process(self, file):
+        """Input.process(...) -> [ Dataset, ... ]
+
+        Called to process a file and return an array of `Dataset`'s found in it.
+        """
         raise Exception('process() not overloaded')
 
+
     def __init_subclass__(cls):
+        """This method is called when a class is subclassed and it will
+        register the input plugin in `inputs`."""
         global inputs
         if cls.__name__ in inputs:
             raise Exception('Duplicate input module: %s already exists' % cls.__name__)
         inputs[cls.__name__] = cls
 
 
 class Output(object):
+    """Base class of an output plugin"""
+
+
     def process(self, datasets):
+        """Output.process([ Dataset, ... ])
+
+        Called to output the `Dataset`'s in the given array."""
         raise Exception('process() not overloaded')
 
+
     def __init__(self, opts):
+        """instance = Output({ 'opt': value, ... })
+
+        Called to create an instance of the output plugin, will get a dict
+        with options provided on command line."""
         pass
 
+
     def __init_subclass__(cls):
+        """This method is called when a class is subclassed and it will
+        register the output plugin in `outputs`."""
         global outputs
         if cls.__name__ in outputs:
             raise Exception('Duplicate output module: %s already exists' % cls.__name__)
         outputs[cls.__name__] = cls
 
 
 class Generator(object):
+    """Base class of a generator plugin"""
+
+
     def process(self, datasets):
+        """Generator.process([ Dataset, ... ]) -> [ Dataset, ... ]
+
+        Called to generate additional `Dataset`'s based on the given array
+        of `Dataset`'s."""
         raise Exception('process() not overloaded')
 
+
     def __init__(self, opts):
+        """instance = Generator({ 'opt': value, ... })
+
+        Called to create an instance of the generator plugin, will get a dict
+        with options provided on command line."""
         pass
 
+
     def __init_subclass__(cls):
+        """This method is called when a class is subclassed and it will
+        register the generator plugin in `generators`."""
         global generators
         if cls.__name__ in generators:
             raise Exception('Duplicate generator module: %s already exists' % cls.__name__)
         generators[cls.__name__] = cls
 
 
 class Transformer(object):
+    """Base class of a transformer plugin"""
+
+
     def process(self, datasets):
+        """Transformer.process([ Dataset, ... ])
+
+        Called to do transformation of the given `Dataset`'s, as in modifying
+        them directly."""
         raise Exception('process() not overloaded')
 
+
     def __init__(self, opts):
+        """instance = Transformer({ 'opt': value, ... })
+
+        Called to create an instance of the transformer plugin, will get a dict
+        with options provided on command line."""
         pass
 
+
     def __init_subclass__(cls):
+        """This method is called when a class is subclassed and it will
+        register the transformer plugin in `transformers`."""
         global transformers
         if cls.__name__ in transformers:
             raise Exception('Duplicate transformer module: %s already exists' % cls.__name__)
         transformers[cls.__name__] = cls
 
 
 def main():
+    """Called when running `dsc-datatool`."""
     def iter_namespace(ns_pkg):
         return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
 

dsc_datatool/generator/client_ports_count.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.generator.client_ports_count
+
+Not implemented.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 from dsc_datatool import Generator

dsc_datatool/generator/client_subnet_authority.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.generator.client_subnet_authority
+
+See `man dsc-datatool-generator client_subnet_authority`.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import csv

dsc_datatool/generator/client_subnet_country.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.generator.client_subnet_country
+
+See `man dsc-datatool-generator client_subnet_country`.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import maxminddb

dsc_datatool/input/dat.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.input.dat
+
+Input plugin to generate `Dataset`'s from DSC DAT files.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import re

dsc_datatool/input/xml.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.input.xml
+
+Input plugin to generate `Dataset`'s from DSC XML files.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import logging

dsc_datatool/output/influxdb.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.output.influxdb
+
+See `man dsc-datatool-output influxdb`.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import re

dsc_datatool/transformer/labler.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.transformer.labler
+
+See `man dsc-datatool-transformer labler`.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import yaml

dsc_datatool/transformer/net_remap.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.transformer.net_remap
+
+See `man dsc-datatool-transformer netremap`.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import ipaddress

dsc_datatool/transformer/re_ranger.py

@@ -1,10 +1,10 @@
-"""
-    dsc_datatool
-    ~~~~~~~~~~~~
+"""dsc_datatool.transformer.re_ranger
+
+See `man dsc-datatool-transformer reranger`.
 
-    DSC datatool
+Part of dsc_datatool.
 
-    :copyright: 2020 OARC, Inc.
+:copyright: 2020 OARC, Inc.
 """
 
 import re