Fix coverage collection for skipped/incomplete tests

src/Phpunit/TestCase.php

@@ -6,6 +6,12 @@
 
 use Atk4\Core\WarnDynamicPropertyTrait;
 use PHPUnit\Framework\TestCase as BaseTestCase;
+use PHPUnit\Framework\TestResult;
+use PHPUnit\Runner\AfterTestHook;
+use PHPUnit\Runner\BaseTestRunner;
+use PHPUnit\Runner\TestListenerAdapter;
+use PHPUnit\Util\Test as TestUtil;
+use SebastianBergmann\CodeCoverage\CodeCoverage;
 
 /**
  * Generic TestCase for PHPUnit tests for ATK4 repos.
@@ -66,8 +72,57 @@ protected function tearDown(): void
 
         // fix coverage when no assertion is expected
         // https://github.com/sebastianbergmann/phpunit/pull/5010
-        if ($this->getNumAssertions() === 0 && $this->doesNotPerformAssertions()) {
-            $this->getTestResultObject()->beStrictAboutTestsThatDoNotTestAnything(false);
+        if ($this->getStatus() === BaseTestRunner::STATUS_PASSED
+            && $this->getNumAssertions() === 0 && $this->doesNotPerformAssertions()
+            && $this->getTestResultObject()->isStrictAboutTestsThatDoNotTestAnything()
+        ) {
+            $testResult = $this->getTestResultObject();
+            $afterHookTest = new class($testResult) implements AfterTestHook {
+                /** @var TestResult */
+                public $testResult;
+
+                public function __construct(TestResult $testResult)
+                {
+                    $this->testResult = $testResult;
+                }
+
+                public function executeAfterTest(string $test, float $time): void
+                {
+                    $this->testResult->beStrictAboutTestsThatDoNotTestAnything(true);
+                }
+            };
+            $alreadyAdded = false;
+            foreach (\Closure::bind(fn () => $testResult->listeners, null, TestResult::class)() as $listener) { // @phpstan-ignore-line
+                if ($listener instanceof TestListenerAdapter) {
+                    foreach (\Closure::bind(fn () => $listener->hooks, null, TestListenerAdapter::class)() as $hook) {
+                        if (get_class($hook) === get_class($afterHookTest)) {
+                            $alreadyAdded = true;
+                        }
+                    }
+                }
+            }
+            if (!$alreadyAdded) {
+                $testListenerAdapter = new TestListenerAdapter();
+                $testListenerAdapter->add($afterHookTest);
+                $testResult->addListener($testListenerAdapter); // @phpstan-ignore-line
+            }
+            $testResult->beStrictAboutTestsThatDoNotTestAnything(false);
+        }
+
+        // fix coverage for skipped/incomplete tests
+        // based on https://github.com/sebastianbergmann/phpunit/blob/9.5.21/src/Framework/TestResult.php#L830
+        // and https://github.com/sebastianbergmann/phpunit/blob/9.5.21/src/Framework/TestResult.php#L857
+        if (in_array($this->getStatus(), [BaseTestRunner::STATUS_SKIPPED, BaseTestRunner::STATUS_INCOMPLETE], true)) {
+            $coverage = $this->getTestResultObject()->getCodeCoverage();
+            if ($coverage !== null) {
+                $coverageId = \Closure::bind(fn () => $coverage->currentId, null, CodeCoverage::class)();
+                if ($coverageId !== null) { // @phpstan-ignore-line https://github.com/sebastianbergmann/php-code-coverage/pull/923
+                    $linesToBeCovered = TestUtil::getLinesToBeCovered(static::class, $this->getName(false));
+                    $linesToBeUsed = TestUtil::getLinesToBeUsed(static::class, $this->getName(false));
+                    $coverage->stop(true, $linesToBeCovered, $linesToBeUsed);
+                    $coverage->start($coverageId);
+                }
+            }
         }
     }
 

src/WarnDynamicPropertyTrait.php

@@ -17,7 +17,7 @@ protected function warnPropertyDoesNotExist(string $name): void
         $class = static::class;
         try {
             $propRefl = new \ReflectionProperty($class, $name);
-            if (!$propRefl->isPrivate()) {
+            if (!$propRefl->isPrivate() && !$propRefl->isPublic() && !$propRefl->isStatic()) {
                 throw new \Error('Cannot access protected property ' . $class . '::$' . $name);
             }
         } catch (\ReflectionException $e) {

tests/Phpunit/ResultPrinterTest.php

@@ -0,0 +1,81 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Atk4\Core\Tests\Phpunit;
+
+use Atk4\Core\Exception;
+use Atk4\Core\Phpunit\ResultPrinter;
+use Atk4\Core\Phpunit\TestCase;
+use PHPUnit\Framework\ExceptionWrapper;
+use PHPUnit\Framework\TestFailure;
+
+class ResultPrinterTest extends TestCase
+{
+    /**
+     * @param class-string<ResultPrinter> $resultPrinterClass
+     */
+    private function printAndReturnDefectTrace($resultPrinterClass, \Throwable $exception): string
+    {
+        $defect = new TestFailure($this, $exception);
+        $stream = fopen('php://memory', 'w+');
+        $printer = new $resultPrinterClass($stream);
+        \Closure::bind(fn () => $printer->printDefectTrace($defect), null, $resultPrinterClass)();
+        fseek($stream, 0);
+
+        return stream_get_contents($stream);
+    }
+
+    public function testBasic(): void
+    {
+        $innerException = new \Error('Inner Exception');
+        $exception = (new Exception('My exception', 0, $innerException))
+            ->addMoreInfo('x', 'foo')
+            ->addMoreInfo('y', ['bar' => 2.4, [], [[1]]]);
+
+        $resNotWrapped = $this->printAndReturnDefectTrace(ResultPrinter::class, $exception);
+        $this->assertStringContainsString((string) $exception, $resNotWrapped);
+        $this->assertStringContainsString((string) $innerException, $resNotWrapped);
+
+        $staticClass = get_class(new class() {
+            public static int $counter = 0;
+        });
+        if (++$staticClass::$counter > 2) {
+            // allow this test to be run max. twice, new ExceptionWrapper() is leaking memory,
+            // see https://github.com/sebastianbergmann/phpunit/blob/9.5.21/src/Framework/ExceptionWrapper.php#L112
+            // https://github.com/sebastianbergmann/phpunit/pull/5012
+            // @codeCoverageIgnoreStart
+            return;
+            // @codeCoverageIgnoreEnd
+        }
+
+        $res = $this->printAndReturnDefectTrace(ResultPrinter::class, new ExceptionWrapper($exception));
+        $this->assertTrue(strlen($res) < strlen($resNotWrapped));
+        if (\PHP_MAJOR_VERSION < 8) {
+            // phpvfscomposer:// is not correctly filtered from stacktrace
+            // by PHPUnit\Util\Filter::getFilteredStacktrace() method
+            return;
+        }
+        $this->assertSame(
+            <<<'EOF'
+                Atk4\Core\Exception: My exception
+                  x: 'foo'
+                  y: [
+                      'bar': 2.4,
+                      0: [],
+                      1: [
+                          ...
+                        ]
+                    ]
+
+                self.php:32
+
+                Caused by
+                Error: Inner Exception
+
+                self.php:31
+                EOF . "\n", // NL in the string is not parsed by Netbeans, see https://github.com/apache/netbeans/issues/4345
+            str_replace(__FILE__, 'self.php', $res)
+        );
+    }
+}

tests/Phpunit/TestCaseTest.php

@@ -5,11 +5,12 @@
 namespace Atk4\Core\Tests\Phpunit;
 
 use Atk4\Core\Phpunit\TestCase;
+use PHPUnit\Framework\TestCase as PhpunitTestCase;
+use PHPUnit\Runner\BaseTestRunner;
 
 class TestCaseTest extends TestCase
 {
-    /** @var int */
-    private static $providerCallCounter = 0;
+    private static int $providerCallCounter = 0;
 
     private function coverCoverageFromProvider(): void
     {
@@ -20,17 +21,18 @@ private function coverCoverageFromProvider(): void
      * @dataProvider provideProviderCoverage1
      * @dataProvider provideProviderCoverage2
      */
-    public function testProviderCoverage1(string $v): void
+    public function testProviderCoverage(string $v): void
     {
         if ($v === 'y') {
             $this->assertSame(2, self::$providerCallCounter);
         }
-        $this->assertTrue(in_array($v, ['a', 'x', 'y'], true));
+        $this->assertTrue(in_array($v, ['a', 'b', 'x', 'y'], true));
     }
 
     public function provideProviderCoverage1(): \Traversable
     {
         yield ['a'];
+        yield ['b'];
     }
 
     public function provideProviderCoverage2(): \Traversable
@@ -39,4 +41,59 @@ public function provideProviderCoverage2(): \Traversable
         $this->coverCoverageFromProvider();
         yield ['y'];
     }
+
+    /**
+     * @dataProvider provideProviderCoverage1
+     */
+    public function testCoverageImplForDoesNotPerformAssertions(string $v): void
+    {
+        $this->assertFalse($this->doesNotPerformAssertions());
+
+        $staticClass = get_class(new class() {
+            public static int $counter = 0;
+        });
+        if ($v === 'a' && ++$staticClass::$counter > 1) {
+            // allow TestCase::runBare() to be run more than once
+            // @codeCoverageIgnoreStart
+            return;
+            // @codeCoverageIgnoreEnd
+        }
+
+        $this->assertTrue($this->getTestResultObject()->isStrictAboutTestsThatDoNotTestAnything());
+
+        if ($v === 'b') {
+            // make sure TestResult::$beStrictAboutTestsThatDoNotTestAnything is reset
+            // after this test by AfterTestHook hook added by our TestCase
+            return;
+        }
+
+        $testStatusOrig = \Closure::bind(fn () => $this->status, $this, PhpunitTestCase::class)();
+        \Closure::bind(fn () => $this->status = BaseTestRunner::STATUS_PASSED, $this, PhpunitTestCase::class)();
+        try {
+            \Closure::bind(fn () => $this->doesNotPerformAssertions = true, $this, PhpunitTestCase::class)();
+            try {
+                $this->tearDown();
+            } finally {
+                \Closure::bind(fn () => $this->doesNotPerformAssertions = false, $this, PhpunitTestCase::class)();
+            }
+        } finally {
+            \Closure::bind(fn () => $this->status = $testStatusOrig, $this, PhpunitTestCase::class)();
+        }
+
+        $this->assertFalse($this->getTestResultObject()->isStrictAboutTestsThatDoNotTestAnything());
+    }
+
+    /**
+     * @doesNotPerformAssertions
+     */
+    public function testCoverageImplForTestMarkedAsIncomplete(): void
+    {
+        $testStatusOrig = \Closure::bind(fn () => $this->status, $this, PhpunitTestCase::class)();
+        \Closure::bind(fn () => $this->status = BaseTestRunner::STATUS_INCOMPLETE, $this, PhpunitTestCase::class)();
+        try {
+            $this->tearDown();
+        } finally {
+            \Closure::bind(fn () => $this->status = $testStatusOrig, $this, PhpunitTestCase::class)();
+        }
+    }
 }

tests/WarnDynamicPropertyTraitTest.php

@@ -0,0 +1,107 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Atk4\Core\Tests;
+
+use Atk4\Core\Exception as ClassWithWarnDynamicPropertyTrait;
+use Atk4\Core\Phpunit\TestCase;
+
+class WarnDynamicPropertyTraitTest extends TestCase
+{
+    protected function runWithErrorConvertedToException(\Closure $fx): void
+    {
+        set_error_handler(function (int $errno, string $errstr): void {
+            throw new WarnError($errstr);
+        });
+        try {
+            $fx();
+        } finally {
+            restore_error_handler();
+        }
+    }
+
+    public function testIssetException(): void
+    {
+        $this->runWithErrorConvertedToException(fn () => null);
+        $this->runWithErrorConvertedToException(function () {
+            $test = new ClassWithWarnDynamicPropertyTrait();
+
+            $this->expectException(WarnError::class);
+            $this->expectErrorMessage('Undefined property: Atk4\Core\Exception::$xxx');
+            isset($test->xxx); // @phpstan-ignore-line
+        });
+    }
+
+    public function testGetException(): void
+    {
+        $this->runWithErrorConvertedToException(function () {
+            $test = new ClassWithWarnDynamicPropertyTrait();
+
+            $this->expectException(WarnError::class);
+            $this->expectErrorMessage('Undefined property: Atk4\Core\Exception::$xxx');
+            $test->xxx; // @phpstan-ignore-line
+        });
+    }
+
+    public function testSetException(): void
+    {
+        $this->runWithErrorConvertedToException(function () {
+            $test = new ClassWithWarnDynamicPropertyTrait();
+
+            $this->expectException(WarnError::class);
+            $this->expectErrorMessage('Undefined property: Atk4\Core\Exception::$xxx');
+            $test->xxx = 5; // @phpstan-ignore-line
+        });
+    }
+
+    public function testUnsetException(): void
+    {
+        $this->runWithErrorConvertedToException(function () {
+            $test = new ClassWithWarnDynamicPropertyTrait();
+
+            $this->expectException(WarnError::class);
+            $this->expectErrorMessage('Undefined property: Atk4\Core\Exception::$xxx');
+            unset($test->{'xxx'});
+        });
+    }
+
+    public function testGetSetPublicProperty(): void
+    {
+        $test = new ClassWithWarnDynamicPropertyTrait();
+        $this->assertTrue(isset($test->params)); // @phpstan-ignore-line
+        $test->params = ['foo'];
+        $this->assertSame(['foo'], $test->params);
+        unset($test->{'params'});
+
+        $this->runWithErrorConvertedToException(function () use ($test) {
+            $this->expectException(WarnError::class);
+            $this->expectErrorMessage('Undefined property: Atk4\Core\Exception::$params');
+            $test->params; // @phpstan-ignore-line
+        });
+    }
+
+    public function testGetProtectedPropertyException(): void
+    {
+        $test = new ClassWithWarnDynamicPropertyTrait();
+
+        $this->expectException(\Error::class);
+        $this->expectErrorMessage('Cannot access protected property Atk4\Core\Exception::$customExceptionTitle');
+        $test->customExceptionTitle; // @phpstan-ignore-line
+    }
+
+    public function testGetPrivateProperty(): void
+    {
+        $this->runWithErrorConvertedToException(function () {
+            $test = new ClassWithWarnDynamicPropertyTrait();
+
+            $this->expectException(WarnError::class);
+            $this->expectErrorMessage('Undefined property: Atk4\Core\Exception::$trace2');
+            $test->trace2; // @phpstan-ignore-line
+        });
+    }
+}
+
+class WarnError extends \Exception
+{
+}