Merge pull request #44 from buildtesters/moduleloadtest_stats

add support for retrieving  stats from ModuleLoadTest

Author: shahzebsiddiqui

.coveragerc

@@ -12,7 +12,6 @@ exclude_lines =
 
     # Don't complain about missing debug-only code:
     def __repr__
-    if self\.debug
 
     # Don't complain if tests don't hit defensive assertion code:
     raise AssertionError

docs/moduleloadtest.rst

@@ -96,70 +96,77 @@ Next we will show how we can filter modules while testing. Currently, we can fil
 and exclude modules by full canonical module name. This can be useful for site-administrators to tweak how behavior
 of ``ModuleLoadTest`` to their liking.
 
-For example, some sites may have some modules like ``VASP``, ``Matlab``, ``Gaussian`` that can only be loaded
-by a specific unix group because site-administrator want to restrict this software to be loaded by anyone and end
-up running the software which may take up a license seat.
-
-To filter by module names you can pass ``name`` option which is a list of software names to test.
+To filter by module names you can pass ``name`` option which is a list of software names to test. In example, below we will test for all
+modules `gcc` and `autoconf`.
 
 .. code-block:: python
 
-    >>> g = ModuleLoadTest("/mxg-hpc/users/ssi29/easybuild/modules/all",name=["Automake","Bison"])
-    Testing the Following Module Trees: /mxg-hpc/users/ssi29/easybuild/modules/all
+    >>> ModuleLoadTest(name=["gcc","autoconf"],login=True)
+    Testing the Following Module Trees: /Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core:/usr/local/Cellar/lmod/8.6.14/modulefiles/Darwin:/usr/local/Cellar/lmod/8.6.14/modulefiles/Core
     ________________________________________________________________________________
-    PASSED -  Module Name: Automake/1.16.1-GCCcore-8.3.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Automake/1.16.1-GCCcore-8.3.0.lua )
-    PASSED -  Module Name: Automake/1.15.1-GCCcore-6.4.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Automake/1.15.1-GCCcore-6.4.0.lua )
-    PASSED -  Module Name: Bison/3.0.5 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.0.5.lua )
-    PASSED -  Module Name: Bison/3.0.4-GCCcore-7.1.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.0.4-GCCcore-7.1.0.lua )
-    PASSED -  Module Name: Bison/3.0.4 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.0.4.lua )
-    PASSED -  Module Name: Bison/3.3.2 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.3.2.lua )
-    PASSED -  Module Name: Bison/3.2.2-GCCcore-7.4.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.2.2-GCCcore-7.4.0.lua )
-    PASSED -  Module Name: Bison/3.0.4-GCCcore-6.4.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.0.4-GCCcore-6.4.0.lua )
-    PASSED -  Module Name: Bison/3.0.4-GCCcore-8.1.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.0.4-GCCcore-8.1.0.lua )
-    PASSED -  Module Name: Bison/3.0.5-GCCcore-6.4.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.0.5-GCCcore-6.4.0.lua )
-    PASSED -  Module Name: Bison/3.3.2-GCCcore-8.3.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.3.2-GCCcore-8.3.0.lua )
-    PASSED -  Module Name: Bison/3.0.5-GCCcore-8.1.0 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/Bison/3.0.5-GCCcore-8.1.0.lua )
-
-
-Note, when you use ``name`` it will test all modules with the name ``Automake`` and ``Bison`` found in all module trees.
+    PASSED -  Module Name: autoconf/2.69-3yrvwbu ( modulefile=/Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core/autoconf/2.69-3yrvwbu.lua )
+    PASSED -  Module Name: gcc/9.3.0-n7p74fd ( modulefile=/Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core/gcc/9.3.0-n7p74fd.lua )
+    PASSED -  Module Name: gcc/10.2.0-37fmsw7 ( modulefile=/Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core/gcc/10.2.0-37fmsw7.lua )
+
+
+Note, when you use ``name`` it will test all modules that match the name, found in all module trees.
 If you would like to filter and include by a full canonical name you can specify the ``include`` option. Shown below
-we will only test module ``CUDA/10.0.130``.
+we will test module ``gcc/9.3.0-n7p74fd``.
 
 .. code-block:: python
 
-    >>> h = ModuleLoadTest("/mxg-hpc/users/ssi29/easybuild/modules/all",include=["CUDA/10.0.130"])
-    Testing the Following Module Trees: /mxg-hpc/users/ssi29/easybuild/modules/all
+    >>> ModuleLoadTest(include=['gcc/9.3.0-n7p74fd'],login=True)
+    Testing the Following Module Trees: /Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core:/usr/local/Cellar/lmod/8.6.14/modulefiles/Darwin:/usr/local/Cellar/lmod/8.6.14/modulefiles/Core
     ________________________________________________________________________________
-    PASSED -  Module Name: CUDA/10.0.130 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/CUDA/10.0.130.lua )
+    PASSED -  Module Name: gcc/9.3.0-n7p74fd ( modulefile=/Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core/gcc/9.3.0-n7p74fd.lua )
 
 Likewise, we can exclude module by full canonical name using the ``exclude`` argument which is a list of module names. In
 example below we test the module tree ``"/usr/share/lmod/lmod/modulefiles/Core"`` which comes with ``lmod`` and ``settarg``
 typically found when installing Lmod. In the second example we exclude ``lmod`` from the module test.
 
 .. code-block:: python
 
-    >>> a = ModuleLoadTest("/usr/share/lmod/lmod/modulefiles/Core")
+    >>> ModuleLoadTest("/usr/share/lmod/lmod/modulefiles/Core")
     Testing the Following Module Trees: /usr/share/lmod/lmod/modulefiles/Core
     ________________________________________________________________________________
     PASSED -  Module Name: lmod ( modulefile=/usr/share/lmod/lmod/modulefiles/Core/lmod.lua )
     PASSED -  Module Name: settarg ( modulefile=/usr/share/lmod/lmod/modulefiles/Core/settarg.lua )
 
-    >>> b = ModuleLoadTest("/usr/share/lmod/lmod/modulefiles/Core",exclude=["lmod"])
+    >>> ModuleLoadTest("/usr/share/lmod/lmod/modulefiles/Core",exclude=["lmod"])
     Testing the Following Module Trees: /usr/share/lmod/lmod/modulefiles/Core
     ________________________________________________________________________________
     PASSED -  Module Name: settarg ( modulefile=/usr/share/lmod/lmod/modulefiles/Core/settarg.lua )
 
 
-If you pass ``include`` and ``exclude`` to *ModuleLoadTest*, then *include* will take precedence and *exclude*
-list will be ignored. Since these are mutually exclusive options use either arguments but don't use both at same time.
+If you pass ``include`` and ``exclude`` to *ModuleLoadTest*, we perform the include operation first by filtering files
+followed by excluding files from list. Therefore, if you run the following example, where we include ``lmod`` and ``settarg``
+but also exclude ``lmod``, we will see that ``lmod`` is not tested.
 
 .. code-block:: python
 
-    >>> a = ModuleLoadTest(include=["CUDA/10.0.130","Bison/3.0.4"],exclude=["lmod"])
-    Testing the Following Module Trees: /mxg-hpc/users/ssi29/easybuild-HMNS/modules/all/Core:/mxg-hpc/users/ssi29/spack/modules/linux-rhel7-x86_64/Core:/mxg-hpc/users/ssi29/easybuild/modules/all:/etc/modulefiles:/usr/share/modulefiles:/usr/share/modulefiles/Linux:/usr/share/modulefiles/Core:/usr/share/lmod/lmod/modulefiles/Core
+    >>>  ModuleLoadTest(name=["lmod","settarg"],exclude=['lmod'], login=True)
+    Testing the Following Module Trees: /Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core:/usr/local/Cellar/lmod/8.6.14/modulefiles/Darwin:/usr/local/Cellar/lmod/8.6.14/modulefiles/Core
+    ________________________________________________________________________________
+    PASSED -  Module Name: settarg ( modulefile=/usr/local/Cellar/lmod/8.6.14/modulefiles/Core/settarg.lua )
+
+Get Test Results
+-----------------
+
+The **get_results** method returns a dictionary of results which can be useful if you want to know number of pass and failed test.
+In this next example, we will invoke the class and call ``get_results`` method which returns the test results.
+
+.. code-block:: python
+
+    >>> test = ModuleLoadTest(name=["lmod","settarg"], login=True)
+    Testing the Following Module Trees: /Users/siddiq90/projects/spack/share/spack/lmod/darwin-catalina-x86_64/Core:/usr/local/Cellar/lmod/8.6.14/modulefiles/Darwin:/usr/local/Cellar/lmod/8.6.14/modulefiles/Core
     ________________________________________________________________________________
-    PASSED -  Module Name: Bison/3.0.4 ( modulefile=/mxg-hpc/users/ssi29/easybuild-HMNS/modules/all/Core/Bison/3.0.4.lua )
-    PASSED -  Module Name: CUDA/10.0.130 ( modulefile=/mxg-hpc/users/ssi29/easybuild/modules/all/CUDA/10.0.130.lua )
+    PASSED -  Module Name: lmod ( modulefile=/usr/local/Cellar/lmod/8.6.14/modulefiles/Core/lmod.lua )
+    PASSED -  Module Name: settarg ( modulefile=/usr/local/Cellar/lmod/8.6.14/modulefiles/Core/settarg.lua )
+    >>> test.get_results()
+    {'passed': 2, 'failed': 0, 'total': 2, 'rate': 1.0}
+
+You can use this method to help write your test, for instance you may want to write a test that checks if 'rate' is 1.0 which indicates
+**100%** pass, or you can set a threshold such as 0.9 which indicates 90% pass.
 
 Test Modules in Login Shell
 ----------------------------
@@ -188,3 +195,20 @@ actual command using login shell.
     [DEBUG] Return Code: 0
     PASSED -  Module Name: settarg ( modulefile=/usr/share/lmod/lmod/modulefiles/Core/settarg.lua )
 
+
+Edge Cases
+-----------
+
+If you specify ``count=0``, which controls number of modules to test, you will get an error during testing with the following message
+
+.. code-block:: python
+
+    >>> ModuleLoadTest(count=0)
+    Please specify a number greater than 0 in order to test modules
+
+Alternatively, if you specify a search criteria or MODULEPATH to tree where no modules are detected, you will get an error message
+
+.. code-block:: python
+
+    >>> ModuleLoadTest(include=['foo'])
+    Unable to test any modules either no modules were detected or search criteria was too restrictive

lmod/moduleloadtest.py

@@ -1,4 +1,5 @@
 import os
+import sys
 from lmod.module import Module
 from lmod.spider import Spider
 
@@ -66,7 +67,13 @@ def __init__(
         self.name = name
         self.include = include
         self.exclude = exclude
-        filter_modules = None
+
+        self.failed = 0
+        self.passed = 0
+        self.modulecount = 0
+
+        if self.count < 1:
+            sys.exit("Please specify a number greater than 0 in order to test modules")
 
         spider_cmd = Spider(self.tree)
         module_dict, modules = (
@@ -75,15 +82,18 @@ def __init__(
         )
 
         if self.include:
-            filter_modules = set(self.include).intersection(modules)
+            modules = [module for module in modules if module in self.include]
 
         # only do exclusion if include list is not specified
-        if self.exclude and not self.include:
-            filter_modules = set(modules).difference(self.exclude)
-
-        modules = filter_modules or modules
-
-        modulecount = 0
+        if self.exclude:
+            for name in self.exclude:
+                if name in modules:
+                    modules.remove(name)
+
+        if not modules:
+            sys.exit(
+                "Unable to test any modules either no modules were detected or search criteria was too restrictive"
+            )
 
         print(f"Testing the Following Module Trees: {self.tree}")
         print("{:_<80}".format(""))
@@ -108,13 +118,30 @@ def __init__(
                 print(
                     f"PASSED -  Module Name: {module_name} ( modulefile={modulefile} )"
                 )
+                self.passed += 1
             else:
                 print(
                     f"FAILED -  Module Name: {module_name} ( modulefile={modulefile} )"
                 )
+                self.failed += 1
 
-            modulecount += 1
+            self.modulecount += 1
 
             # terminate module load test once we have tested up to specified count
-            if self.count <= modulecount:
+            if self.count <= self.modulecount:
                 return
+
+    def get_results(self):
+        """This method returns a dictionary of test results
+
+        :return: dictionary of test results
+        :rtype: dict
+        """
+        return {
+            "passed": self.passed,
+            "failed": self.failed,
+            "total": self.modulecount,
+            "rate": self.passed / self.modulecount
+            if self.modulecount
+            else float("nan"),
+        }

tests/test_moduleloadtest.py

@@ -1,4 +1,5 @@
 import os
+import pytest
 from lmod.moduleloadtest import ModuleLoadTest
 
 
@@ -16,13 +17,17 @@ def test_filter_exclude(self):
         ModuleLoadTest(exclude=["lmod"])
 
     def test_filter_include_exclude(self):
-        ModuleLoadTest(include=["lmod"], exclude=["lmod"])
+        with pytest.raises(SystemExit):
+            ModuleLoadTest(include=["lmod"], exclude=["lmod"])
 
     def test_filter_name(self):
         ModuleLoadTest(name=["lmod", "settarg"])
 
     def test_by_count(self):
-        ModuleLoadTest(count=1)
+        m = ModuleLoadTest(count=1)
+        m.get_results()
+        with pytest.raises(SystemExit):
+            ModuleLoadTest(count=0)
 
     def test_debug(self):
         ModuleLoadTest(debug=True)

tests/test_modules.py

@@ -40,7 +40,7 @@ def test_version(self):
         a.version()
 
     def test_avail(self):
-        a = Module()
+        a = Module(debug=True)
         a.avail()
         a.avail("lmod")
 
@@ -125,7 +125,7 @@ def test_spider(self):
             a.spider(1)
 
     def test_checkSyntax(self):
-        a = Module("lmod")
+        a = Module("lmod", debug=True)
         assert 0 == a.checkSyntax()
 
         with pytest.raises(TypeError):