Merge pull request #6351 from cjerdonek/tighten-test-stderr-checks

Tighten up the test suite's stderr checks

Author: cjerdonek

tests/conftest.py

@@ -303,7 +303,7 @@ def script(tmpdir, virtualenv, deprecated_python):
         assert_no_temp=True,
 
         # Deprecated python versions produce an extra deprecation warning
-        pip_expect_stderr=deprecated_python,
+        pip_expect_warning=deprecated_python,
     )
 
 

tests/functional/test_install.py

@@ -1474,8 +1474,7 @@ def test_install_conflict_results_in_warning(script, data):
 
     # Then install an incorrect version of the dependency
     result2 = script.pip(
-        'install', '--no-index', pkgB_path,
-        expect_stderr=True,
+        'install', '--no-index', pkgB_path, allow_stderr_error=True,
     )
     assert "pkga 1.0 has requirement pkgb==1.0" in result2.stderr, str(result2)
     assert "Successfully installed pkgB-2.0" in result2.stdout, str(result2)

tests/lib/__init__.py

@@ -378,7 +378,7 @@ def __init__(self, base_path, *args, **kwargs):
 
         # Whether all pip invocations should expect stderr
         # (useful for Python version deprecation)
-        self.pip_expect_stderr = kwargs.pop('pip_expect_stderr', None)
+        self.pip_expect_warning = kwargs.pop('pip_expect_warning', None)
 
         # Call the TestFileEnvironment __init__
         super(PipTestEnvironment, self).__init__(base_path, *args, **kwargs)
@@ -423,9 +423,9 @@ def run(self, *args, **kw):
         :param allow_stderr_error: whether a logged error is allowed in
             stderr.  Passing True for this argument implies
             `allow_stderr_warning` since warnings are weaker than errors.
-        :param expect_stderr: allow any stderr (equivalent to passing
-            `allow_stderr_error`).  This argument is an abbreviated version
-            of `allow_stderr_error` and is also kept for backwards
+        :param expect_stderr: whether to allow warnings in stderr (equivalent
+            to `allow_stderr_warning`).  This argument is an abbreviated
+            version of `allow_stderr_warning` and is also kept for backwards
             compatibility.
         """
         if self.verbose:
@@ -443,9 +443,22 @@ def run(self, *args, **kw):
         allow_stderr_error = kw.pop('allow_stderr_error', None)
         allow_stderr_warning = kw.pop('allow_stderr_warning', None)
 
-        if kw.get('expect_error') or kw.get('expect_stderr'):
+        if kw.get('expect_error'):
             # Then default to allowing logged errors.
+            if allow_stderr_error is not None and not allow_stderr_error:
+                raise RuntimeError(
+                    'cannot pass allow_stderr_error=False with '
+                    'expect_error=True'
+                )
             allow_stderr_error = True
+        elif kw.get('expect_stderr'):
+            # Then default to allowing logged warnings.
+            if allow_stderr_warning is not None and not allow_stderr_warning:
+                raise RuntimeError(
+                    'cannot pass allow_stderr_warning=False with '
+                    'expect_stderr=True'
+                )
+            allow_stderr_warning = True
 
         # Pass expect_stderr=True to allow any stderr.  We do this because
         # we do our checking of stderr further on in check_stderr().
@@ -460,14 +473,8 @@ def run(self, *args, **kw):
         return TestPipResult(result, verbose=self.verbose)
 
     def pip(self, *args, **kwargs):
-        if self.pip_expect_stderr:
-            kwargs['expect_stderr'] = True
-        # On old versions of Python, urllib3/requests will raise a warning
-        # about the lack of an SSLContext. Expect it when running commands
-        # that will touch the outside world.
-        if (pyversion_tuple < (2, 7, 9) and
-                args and args[0] in ('search', 'install', 'download')):
-            kwargs['expect_stderr'] = True
+        if self.pip_expect_warning:
+            kwargs['allow_stderr_warning'] = True
         if kwargs.pop('use_module', True):
             exe = 'python'
             args = ('-m', 'pip') + args

tests/lib/test_lib.py

@@ -139,8 +139,33 @@ def test_run__unexpected_stderr(self, script, prefix, expected_start):
         with assert_error_startswith(RuntimeError, expected_start):
             self.run_stderr_with_prefix(script, prefix)
 
+    def test_run__allow_stderr_error_false_error_with_expect_error(
+        self, script,
+    ):
+        """
+        Test passing allow_stderr_error=False with expect_error=True.
+        """
+        expected_start = (
+            'cannot pass allow_stderr_error=False with expect_error=True'
+        )
+        with assert_error_startswith(RuntimeError, expected_start):
+            script.run('python', allow_stderr_error=False, expect_error=True)
+
+    def test_run__allow_stderr_warning_false_error_with_expect_stderr(
+        self, script,
+    ):
+        """
+        Test passing allow_stderr_warning=False with expect_stderr=True.
+        """
+        expected_start = (
+            'cannot pass allow_stderr_warning=False with expect_stderr=True'
+        )
+        with assert_error_startswith(RuntimeError, expected_start):
+            script.run(
+                'python', allow_stderr_warning=False, expect_stderr=True,
+            )
+
     @pytest.mark.parametrize('arg_name', (
-        'expect_stderr',
         'expect_error',
         'allow_stderr_error',
     ))