MTA-1276: Wrong tests number is displayed for functional tests on Bamboo

orykh · orykh · commit 97808706b8e2 · 2015-02-17T18:09:22.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,4 @@
+* Fixed wrong tests number is displayed for functional tests on Bamboo
 * Fixed fatal error after create screenshot or report on Windows
 * Implemented mechanism of applying 3rd party credentials
 * Fixed setValue() method for StrictselectElement class
diff --git a/Magento/Mtf/System/JUnit.php b/Magento/Mtf/System/JUnit.php
@@ -52,44 +52,49 @@ public function __construct($out = null, $logIncompleteSkipped = false)
     public function endTest(\PHPUnit_Framework_Test $test, $time)
     {
         if (!$test instanceof \PHPUnit_Framework_Warning) {
-            if ($this->attachCurrentTestCase) {
-                if ($test instanceof \PHPUnit_Framework_TestCase) {
-                    $numAssertions = $test->getNumAssertions();
-                    $this->testSuiteAssertions[$this->testSuiteLevel] += $numAssertions;
+            if ($test instanceof \PHPUnit_Framework_TestCase) {
+                $numAssertions = $test->getNumAssertions();
+                $this->testSuiteAssertions[$this->testSuiteLevel] += $numAssertions;
 
-                    $this->currentTestCase->setAttribute('assertions', $numAssertions);
-                }
+                $this->currentTestCase->setAttribute('assertions', $numAssertions);
+            }
 
-                $this->currentTestCase->setAttribute(
-                    'time',
-                    sprintf('%F', $time)
-                );
+            $this->currentTestCase->setAttribute(
+                'time',
+                sprintf('%F', $time)
+            );
+
+            $class = new \ReflectionClass($test);
 
-                $class = new \ReflectionClass($test);
+            $xpath = new \DOMXPath($this->document);
 
-                $xpath = new \DOMXPath($this->document);
-                $query = '//testsuite[@name="' . $class->name . '"]';
-                $entries = $xpath->query($query);
+            $queryForRemoveEmptyTestSuite = '//testsuite/testsuite/testsuite/testsuite';
+            $entriesForRemoveEmptyTestSuite = $xpath->query($queryForRemoveEmptyTestSuite)->item(0);
+            if ($entriesForRemoveEmptyTestSuite) {
+                $entriesForRemoveEmptyTestSuite->parentNode->removeChild($entriesForRemoveEmptyTestSuite);
+            }
+
+            $query = '//testsuite[@name="' . $class->name . '"]';
+            $entries = $xpath->query($query);
 
-                $entries->item(0)->appendChild($this->currentTestCase);
+            $entries->item(0)->appendChild($this->currentTestCase);
 
-                $entries->item(0)->setAttribute("tests", $entries->item(0)->childNodes->length);
+            $entries->item(0)->setAttribute("tests", $entries->item(0)->childNodes->length);
 
-                $errors = $xpath->query('//testsuite[@name="' . $class->name . '"]//error')->length;
-                $entries->item(0)->setAttribute("errors", $errors);
+            $errors = $xpath->query('//testsuite[@name="' . $class->name . '"]//error')->length;
+            $entries->item(0)->setAttribute("errors", $errors);
 
-                $errors = $xpath->query('//testsuite[@name="' . $class->name . '"]//failure')->length;
-                $entries->item(0)->setAttribute("failures", $errors);
+            $errors = $xpath->query('//testsuite[@name="' . $class->name . '"]//failure')->length;
+            $entries->item(0)->setAttribute("failures", $errors);
 
-                $entries->item(0)->setAttribute("time", $entries->item(0)->getAttribute("time") + $time);
+            $entries->item(0)->setAttribute("time", $entries->item(0)->getAttribute("time") + $time);
 
-                if (method_exists($test, 'hasOutput') && $test->hasOutput()) {
-                    $systemOut = $this->document->createElement('system-out');
-                    $systemOut->appendChild(
-                        $this->document->createTextNode($test->getActualOutput())
-                    );
-                    $this->currentTestCase->appendChild($systemOut);
-                }
+            if (method_exists($test, 'hasOutput') && $test->hasOutput()) {
+                $systemOut = $this->document->createElement('system-out');
+                $systemOut->appendChild(
+                    $this->document->createTextNode($test->getActualOutput())
+                );
+                $this->currentTestCase->appendChild($systemOut);
             }
         }
 
diff --git a/Magento/Mtf/TestCase/Injectable.php b/Magento/Mtf/TestCase/Injectable.php
@@ -124,6 +124,16 @@ public function setVariationName($variationName)
         $this->variationName = $variationName;
     }
 
+    /**
+     * Get variation name
+     *
+     * @return string
+     */
+    public function getVariationName()
+    {
+        return $this->variationName;
+    }
+
     /**
      * Run with Variations Iterator
      *
diff --git a/Magento/Mtf/TestRunner/Process/PHPUnitUtils.php b/Magento/Mtf/TestRunner/Process/PHPUnitUtils.php
@@ -25,7 +25,6 @@
 
 /**
  * Class Process
- *
  */
 class PHPUnitUtils extends \PHPUnit_Util_PHP
 {
@@ -38,9 +37,7 @@ class PHPUnitUtils extends \PHPUnit_Util_PHP
      */
     public function runJob($job, array $settings = [])
     {
-        throw new \PHPUnit_Framework_Exception(
-            'Should not call this method.'
-        );
+        throw new \PHPUnit_Framework_Exception('Should not call this method.');
     }
 
     /**
@@ -53,7 +50,7 @@ public function runJob($job, array $settings = [])
      */
     protected function process($pipe, $job)
     {
-        /* do nothing */
+        //
     }
 
     /**
@@ -76,11 +73,7 @@ public function processChildResult(
         $time = 0;
 
         if (!empty($stderr)) {
-            $result->addError(
-                $test,
-                new \PHPUnit_Framework_Exception(trim($stderr)),
-                $time
-            );
+            $result->addError($test, new \PHPUnit_Framework_Exception(trim($stderr)), $time);
         } else {
             set_error_handler(
                 function ($errno, $errstr, $errfile, $errline) {
@@ -98,11 +91,7 @@ function ($errno, $errstr, $errfile, $errline) {
                 restore_error_handler();
                 $childResult = false;
 
-                $result->addError(
-                    $test,
-                    new \PHPUnit_Framework_Exception(trim($stdout), 0, $e),
-                    $time
-                );
+                $result->addError($test, new \PHPUnit_Framework_Exception(trim($stdout), 0, $e), $time);
             }
 
             if ($childResult !== false) {
@@ -116,9 +105,7 @@ function ($errno, $errstr, $errfile, $errline) {
                 $childResult = $childResult['result'];
 
                 if ($result->getCollectCodeCoverageInformation()) {
-                    $result->getCodeCoverage()->merge(
-                        $childResult->getCodeCoverage()
-                    );
+                    $result->getCodeCoverage()->merge($childResult->getCodeCoverage());
                 }
 
                 $time = $childResult->time();
@@ -129,38 +116,43 @@ function ($errno, $errstr, $errfile, $errline) {
                 $failures = $childResult->failures();
 
                 if (!empty($notImplemented)) {
-                    $result->addError(
-                        $test,
-                        $this->getException($notImplemented[0]),
-                        $time
-                    );
-                } elseif (!empty($risky)) {
-                    $result->addError(
-                        $test,
-                        $this->getException($risky[0]),
-                        $time
-                    );
-                } elseif (!empty($skipped)) {
-                    $result->addError(
-                        $test,
-                        $this->getException($skipped[0]),
-                        $time
-                    );
-                } elseif (!empty($errors)) {
-                    foreach ($errors as $error) {
-                        $result->addError(
-                            $test,
-                            $this->getException($error),
-                            $time
-                        );
+                    foreach ($notImplemented as $variation => $error) {
+                        $name = $test->getName();
+                        $test->setName($name . " variation $variation");
+                        $result->addError($test, $this->getException($error), $time);
+                        $test->setName($name);
                     }
-                } elseif (!empty($failures)) {
-                    foreach ($failures as $failure) {
-                        $result->addFailure(
-                            $test,
-                            $this->getException($failure),
-                            $time
-                        );
+                }
+                if (!empty($risky)) {
+                    foreach ($risky as $variation => $error) {
+                        $name = $test->getName();
+                        $test->setName($name . " variation $variation");
+                        $result->addError($test, $this->getException($error), $time);
+                        $test->setName($name);
+                    }
+                }
+                if (!empty($skipped)) {
+                    foreach ($skipped as $variation => $error) {
+                        $name = $test->getName();
+                        $test->setName($name . " variation $variation");
+                        $result->addError($test, $this->getException($error), $time);
+                        $test->setName($name);
+                    }
+                }
+                if (!empty($errors)) {
+                    foreach ($errors as $variation => $error) {
+                        $name = $test->getName();
+                        $test->setName($name . " variation $variation");
+                        $result->addError($test, $this->getException($error), $time);
+                        $test->setName($name);
+                    }
+                }
+                if (!empty($failures)) {
+                    foreach ($failures as $variation => $failure) {
+                        $name = $test->getName();
+                        $test->setName($name . " variation $variation");
+                        $result->addFailure($test, $this->getException($failure), $time);
+                        $test->setName($name);
                     }
                 }
             }
@@ -189,11 +181,7 @@ public function getException(\PHPUnit_Framework_TestFailure $error)
             }
 
             $exception = new \PHPUnit_Framework_SyntheticError(
-                sprintf(
-                    '%s: %s',
-                    $exceptionArray['_PHP_Incomplete_Class_Name'],
-                    $exceptionArray['message']
-                ),
+                sprintf('%s: %s', $exceptionArray['_PHP_Incomplete_Class_Name'], $exceptionArray['message']),
                 $exceptionArray['code'],
                 $exceptionArray['file'],
                 $exceptionArray['line'],
diff --git a/Magento/Mtf/TestRunner/Process/TestResult.php b/Magento/Mtf/TestRunner/Process/TestResult.php
@@ -28,6 +28,7 @@
 use Magento\Mtf\TestRunner\Process\Exception\Skipped;
 use Magento\Mtf\TestRunner\Process\Exception\Risky;
 use Magento\Mtf\TestRunner\Process\Exception\Incomplete;
+use Magento\Mtf\TestCase\Injectable;
 
 /**
  * Class ProcessTestResult
@@ -48,21 +49,105 @@ class TestResult extends \PHPUnit_Framework_TestResult
      */
     public function addError(\PHPUnit_Framework_Test $test, \Exception $e, $time)
     {
-        parent::addError($test, $this->wrapException($e), $time);
+        $variation = 0;
+        if ($test instanceof Injectable) {
+            $variation = $test->getVariationName();
+        }
+
+        if ($e instanceof \PHPUnit_Framework_RiskyTest) {
+            $this->risky[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+
+            $notifyMethod = 'addRiskyTest';
+
+            if ($this->stopOnRisky) {
+                $this->stop();
+            }
+        } elseif ($e instanceof \PHPUnit_Framework_IncompleteTest) {
+            $this->notImplemented[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+
+            $notifyMethod = 'addIncompleteTest';
+
+            if ($this->stopOnIncomplete) {
+                $this->stop();
+            }
+        } elseif ($e instanceof \PHPUnit_Framework_SkippedTest) {
+            $this->skipped[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+            $notifyMethod = 'addSkippedTest';
+
+            if ($this->stopOnSkipped) {
+                $this->stop();
+            }
+        } else {
+            $this->errors[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+            $notifyMethod = 'addError';
+
+            if ($this->stopOnError || $this->stopOnFailure) {
+                $this->stop();
+            }
+        }
+
+        foreach ($this->listeners as $listener) {
+            $listener->$notifyMethod($test, $e, $time);
+        }
+
+        $this->lastTestFailed = true;
+        $this->time += $time;
     }
 
     /**
      * Adds a failure to the list of failures.
      * The passed in exception caused the failure.
      *
-     * @param \PHPUnit_Framework_Test                 $test
+     * @param \PHPUnit_Framework_Test $test
      * @param \PHPUnit_Framework_AssertionFailedError $e
-     * @param float                                  $time
+     * @param float $time
      * @return void
      */
     public function addFailure(\PHPUnit_Framework_Test $test, \PHPUnit_Framework_AssertionFailedError $e, $time)
     {
-        parent::addFailure($test, $this->wrapException($e), $time);
+        $variation = null;
+        if ($test instanceof \Magento\Mtf\TestCase\Functional) {
+            $variation = $test->getVariationName();
+        }
+
+        if ($e instanceof \PHPUnit_Framework_RiskyTest) {
+            $this->risky[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+
+            $notifyMethod = 'addRiskyTest';
+
+            if ($this->stopOnRisky) {
+                $this->stop();
+            }
+        } elseif ($e instanceof \PHPUnit_Framework_IncompleteTest) {
+            $this->notImplemented[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+
+            $notifyMethod = 'addIncompleteTest';
+
+            if ($this->stopOnIncomplete) {
+                $this->stop();
+            }
+        } elseif ($e instanceof \PHPUnit_Framework_SkippedTest) {
+            $this->skipped[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+            $notifyMethod = 'addSkippedTest';
+
+            if ($this->stopOnSkipped) {
+                $this->stop();
+            }
+        } else {
+            $this->failures[$variation] = new \PHPUnit_Framework_TestFailure($test, $e);
+            $notifyMethod = 'addFailure';
+
+            if ($this->stopOnFailure) {
+                $this->stop();
+            }
+        }
+
+        foreach ($this->listeners as $listener) {
+            $listener->$notifyMethod($test, $e, $time);
+        }
+
+        $this->lastTestFailed = true;
+        $this->time += $time;
     }
 
     /**

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+* Fixed wrong tests number is displayed for functional tests on Bamboo`
`1`	`2`	`* Fixed fatal error after create screenshot or report on Windows`
`2`	`3`	`* Implemented mechanism of applying 3rd party credentials`
`3`	`4`	`* Fixed setValue() method for StrictselectElement class`