Test: tighten test-skip conditions and lengthen a subprocess sleep

After seeing some AppVeyor failures, I've increased the wait after
starting test HTTP, HTTPS, and proxy servers from 0.5s to 1s, to make
it less likely that tests will fail because the servers weren't done
starting up yet.

After some review comments by @aaaaalbert, I've tightened the logic
in aggregate_tests.py around which tests to skip unless a certain
Python version is running, and added some consistency checks.
This also involved a bit of clarification of comments and variable
names.

Signed-off-by: Sebastien Awwad <sebastien.awwad@gmail.com>

Author: awwad

tests/aggregate_tests.py

@@ -43,40 +43,54 @@
 # 'test_' and end with '.py'.  A shell-style wildcard is used with glob() to
 # match desired filenames.  All the tests matching the pattern will be loaded
 # and run in a test suite.
-tests_list = glob.glob('test_*.py')
+available_tests = glob.glob('test_*.py')
 
 # A dictionary of test modules that should only run in certain python versions.
 # Carefully consider the impact of only testing these in a given version.
 # test_proxy_use.py: uses a proxy that only runs in Python2.7. TUF's
 # compatibility with proxies is not likely to vary based on the Python version
 # in use, so this is OK for now. See comments in that module.
-# The format here is: if major is included, limit version to that listed here.
-# If minor is included, limit version to that listed here. Skip the test if any
-# such listed constraints don't match the python version currently running.
+# The semantics here are: only add to this list the particular tests that are
+# to be run in a single major version or a single minor version. An entry must
+# include major version, and may include minor version.
+# Skip the test if any  such listed constraints don't match the python version
+# currently running.
+# Note that aggregate_tests.py is run for each version of Python that tox is
+# configured to use. Note also that this TUF implementation does not support
+# any Python versions <2.7 or any Python3 versions <3.4.
 VERSION_SPECIFIC_TESTS = {
     'test_proxy_use': {'major': 2, 'minor': 7}} # Run test only if Python2.7
+# Further example:
+#   'test_abc': {'major': 2} # Run test only if Python2
 
-# Remove '.py' from each filename to allow loadTestsFromNames() (called below)
-# to properly load the file as a module.
-tests_without_extension = []
-for test in tests_list:
+# Determine which tests should be run.
+test_modules_to_run = []
+for test in available_tests:
+  # Remove '.py' from each filename to allow loadTestsFromNames() (called below)
+  # to properly load the file as a module.
+  assert test[-3:] == '.py', 'aggregate_tests.py is inconsistent; fix.'
   test = test[:-3]
+
   if test in VERSION_SPECIFIC_TESTS:
-    if 'major' in VERSION_SPECIFIC_TESTS[test] \
-        and sys.version_info.major != VERSION_SPECIFIC_TESTS[test]['major']:
+    # Consistency checks.
+    assert 'major' in VERSION_SPECIFIC_TESTS[test], 'Empty/illogical constraint'
+    for keyword in VERSION_SPECIFIC_TESTS[test]:
+      assert keyword in ['major', 'minor'], 'Unrecognized test constraint'
+
+    if sys.version_info.major != VERSION_SPECIFIC_TESTS[test]['major']:
       continue
-    elif 'minor' in VERSION_SPECIFIC_TESTS[test] \
+    if 'minor' in VERSION_SPECIFIC_TESTS[test] \
         and sys.version_info.minor != VERSION_SPECIFIC_TESTS[test]['minor']:
       continue
-  tests_without_extension.append(test)
+  test_modules_to_run.append(test)
 
 # Randomize the order in which the tests run.  Randomization might catch errors
 # with unit tests that do not properly clean up or restore monkey-patched
 # modules.
-random.shuffle(tests_without_extension)
+random.shuffle(test_modules_to_run)
 
 if __name__ == '__main__':
-  suite = unittest.TestLoader().loadTestsFromNames(tests_without_extension)
+  suite = unittest.TestLoader().loadTestsFromNames(test_modules_to_run)
   all_tests_passed = unittest.TextTestRunner(verbosity=1).run(suite).wasSuccessful()
   if not all_tests_passed:
     sys.exit(1)

tests/test_proxy_use.py

@@ -123,7 +123,8 @@ def setUpClass(cls):
     # start listening before allowing tests to begin, lest we get "Connection
     # refused" errors. On the first test system. 0.1s was too short and 0.15s
     # was long enough. Use 0.5s to be safe, and if issues arise, increase it.
-    time.sleep(0.5)
+    # Observed some occasional AppVeyor failures, so increasing this to 1s.
+    time.sleep(1)