WIP: improve "live" layer performance with materialised views

Neos.ContentGraph.DoctrineDbalAdapter/Tests/Behavior/Features/Bootstrap/DoctrineDbalProjectionIntegrityViolatorTrait.php

@@ -12,29 +12,26 @@
 
 declare(strict_types=1);
 
+use Behat\Gherkin\Node\PyStringNode;
 use Behat\Gherkin\Node\TableNode;
-use Doctrine\DBAL\Connection;
 use Doctrine\DBAL\Exception as DBALException;
 use Doctrine\DBAL\Exception\InvalidArgumentException;
-use Neos\ContentGraph\DoctrineDbalAdapter\ContentGraphTableNames;
+use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Projection\ContentStreamLayer;
 use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\NodeFactory;
 use Neos\ContentGraph\DoctrineDbalAdapter\Tests\Behavior\Features\Bootstrap\Helpers\TestingNodeAggregateId;
 use Neos\ContentRepository\Core\DimensionSpace\DimensionSpacePoint;
 use Neos\ContentRepository\Core\DimensionSpace\OriginDimensionSpacePoint;
 use Neos\ContentRepository\Core\Feature\SubtreeTagging\Dto\SubtreeTag;
 use Neos\ContentRepository\Core\SharedModel\Node\NodeAggregateId;
 use Neos\ContentRepository\Core\SharedModel\Workspace\ContentStreamId;
-use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRTestSuiteRuntimeVariables;
+use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\ProjectionIntegrityViolationDetectionTrait;
+use PHPUnit\Framework\Assert;
 
 /**
  * @internal custom illegal mutations for the Doctrine DBAL content graph adapter to make the projection integrity violation fail
  */
 trait DoctrineDbalProjectionIntegrityViolatorTrait
 {
-    use CRTestSuiteRuntimeVariables;
-
-    private Connection $dbal;
-
     /**
      * @template T of object
      * @param class-string<T> $className
@@ -43,16 +40,19 @@ trait DoctrineDbalProjectionIntegrityViolatorTrait
      */
     abstract private function getObject(string $className): object;
 
-    private function tableNames(): ContentGraphTableNames
+    /**
+     * @When the content stream :contentStreamId was removed without layer cleanup
+     * @throws DBALException
+     */
+    public function theContentStreamWasRemovedWithoutLayerCleanup(string $contentStreamId): void
     {
-        return ContentGraphTableNames::create(
-            $this->currentContentRepository->id
-        );
-    }
+        $this->dbal->delete($this->tableNames()->contentStream(), [
+            'id' => $contentStreamId
+        ]);
 
-    public function setupDbalGraphAdapterIntegrityViolationTrait()
-    {
-        $this->dbal = $this->getObject(Connection::class);
+        $this->dbal->delete($this->tableNames()->contentStreamLayer(), [
+            'contentStreamId' => $contentStreamId,
+        ]);
     }
 
     /**
@@ -92,7 +92,10 @@ public function iAddTheFollowingHierarchyRelation(TableNode $payloadTable): void
         $record = $this->transformDatasetToHierarchyRelationRecord($dataset);
         $this->dbal->insert(
             $this->tableNames()->hierarchyRelation(),
-            $record
+            [
+                'id' => 1000, // FIXME should not be hardcoded if two hierarchies are to be added for testing
+                ...$record
+            ]
         );
     }
 
@@ -108,7 +111,9 @@ public function iChangeTheFollowingHierarchyRelationsParent(TableNode $payloadTa
         unset($record['subtreetags']);
 
         $newParentHierarchyRelation = $this->findHierarchyRelationByIds(
-            ContentStreamId::fromString($dataset['contentStreamId']),
+            $this->requireSingleWriteLayer(
+                ContentStreamId::fromString($dataset['contentStreamId'])
+            ),
             DimensionSpacePoint::fromArray($dataset['dimensionSpacePoint']),
             NodeAggregateId::fromString($dataset['newParentNodeAggregateId'])
         );
@@ -152,14 +157,18 @@ public function iChangeTheFollowingNodesName(TableNode $payloadTable): void
     {
         $dataset = $this->transformPayloadTableToDataset($payloadTable);
 
+        $contentStreamLayer = $this->requireSingleWriteLayer(
+            ContentStreamId::fromString($dataset['contentStreamId'])
+        );
+
         $relationAnchorPoint = $this->dbal->executeQuery(
             'SELECT n.relationanchorpoint FROM ' . $this->tableNames()->node() . ' n
                 JOIN ' . $this->tableNames()->hierarchyRelation() . ' h ON h.childnodeanchor = n.relationanchorpoint
-                WHERE h.contentstreamid = :contentStreamId
+                WHERE h.contentstreamlayer = :contentStreamLayer
                 AND n.nodeaggregateId = :nodeAggregateId
                 AND n.origindimensionspacepointhash = :originDimensionSpacePointHash',
             [
-                'contentStreamId' => $dataset['contentStreamId'],
+                'contentStreamLayer' => $contentStreamLayer->value,
                 'nodeAggregateId' => $dataset['nodeAggregateId'],
                 'originDimensionSpacePointHash' => OriginDimensionSpacePoint::fromArray($dataset['originDimensionSpacePoint'])->hash,
             ]
@@ -185,8 +194,13 @@ public function iSetTheFollowingPosition(TableNode $payloadTable): void
     {
         $dataset = $this->transformPayloadTableToDataset($payloadTable);
         $dimensionSpacePoint = DimensionSpacePoint::fromArray($dataset['dimensionSpacePoint']);
+
+        $contentStreamLayer = $this->requireSingleWriteLayer(
+            ContentStreamId::fromString($dataset['contentStreamId'])
+        );
+
         $record = [
-            'contentstreamid' => $dataset['contentStreamId'],
+            'contentstreamlayer' => $contentStreamLayer->value,
             'dimensionspacepointhash' => $dimensionSpacePoint->hash,
             'childnodeanchor' => $this->findRelationAnchorPointByDataset($dataset)
         ];
@@ -241,7 +255,9 @@ private function transformDatasetToReferenceRelationRecord(array $dataset): arra
         return [
             'name' => $dataset['referenceName'],
             'nodeanchorpoint' => $this->findHierarchyRelationByIds(
-                ContentStreamId::fromString($dataset['contentStreamId']),
+                $this->requireSingleWriteLayer(
+                    ContentStreamId::fromString($dataset['contentStreamId'])
+                ),
                 DimensionSpacePoint::fromArray($dataset['dimensionSpacePoint']),
                 NodeAggregateId::fromString($dataset['sourceNodeAggregateId'])
             )['childnodeanchor'],
@@ -254,25 +270,28 @@ private function transformDatasetToHierarchyRelationRecord(array $dataset): arra
         $dimensionSpacePoint = DimensionSpacePoint::fromArray($dataset['dimensionSpacePoint']);
         $parentNodeAggregateId = TestingNodeAggregateId::fromString($dataset['parentNodeAggregateId']);
         $childAggregateId = TestingNodeAggregateId::fromString($dataset['childNodeAggregateId']);
+        $contentStreamLayer = $this->requireSingleWriteLayer(
+            ContentStreamId::fromString($dataset['contentStreamId'])
+        );
 
         $parentHierarchyRelation = $parentNodeAggregateId->isNonExistent()
             ? null
             : $this->findHierarchyRelationByIds(
-                ContentStreamId::fromString($dataset['contentStreamId']),
+                $contentStreamLayer,
                 $dimensionSpacePoint,
                 NodeAggregateId::fromString($dataset['parentNodeAggregateId'])
             );
 
         $childHierarchyRelation = $childAggregateId->isNonExistent()
             ? null
             : $this->findHierarchyRelationByIds(
-                ContentStreamId::fromString($dataset['contentStreamId']),
+                $contentStreamLayer,
                 $dimensionSpacePoint,
                 NodeAggregateId::fromString($dataset['childNodeAggregateId'])
             );
 
         return [
-            'contentstreamid' => $dataset['contentStreamId'],
+            'contentstreamlayer' => $contentStreamLayer->value,
             'dimensionspacepointhash' => $dimensionSpacePoint->hash,
             'parentnodeanchor' => $parentHierarchyRelation !== null ? $parentHierarchyRelation['childnodeanchor'] : 9999999,
             'childnodeanchor' => $childHierarchyRelation !== null ? $childHierarchyRelation['childnodeanchor'] : 8888888,
@@ -286,14 +305,16 @@ private function findRelationAnchorPointByDataset(array $dataset): int
         $dimensionSpacePoint = DimensionSpacePoint::fromArray($dataset['originDimensionSpacePoint'] ?? $dataset['dimensionSpacePoint']);
 
         return $this->findHierarchyRelationByIds(
-            ContentStreamId::fromString($dataset['contentStreamId']),
+            $this->requireSingleWriteLayer(
+                ContentStreamId::fromString($dataset['contentStreamId'])
+            ),
             $dimensionSpacePoint,
             NodeAggregateId::fromString($dataset['nodeAggregateId'] ?? $dataset['childNodeAggregateId'])
         )['childnodeanchor'];
     }
 
     private function findHierarchyRelationByIds(
-        ContentStreamId $contentStreamId,
+        ContentStreamLayer $contentStreamLayer,
         DimensionSpacePoint $dimensionSpacePoint,
         NodeAggregateId $nodeAggregateId
     ): array {
@@ -302,17 +323,17 @@ private function findHierarchyRelationByIds(
                 FROM ' . $this->tableNames()->node() . ' n
                 INNER JOIN ' . $this->tableNames()->hierarchyRelation() . ' h
                 ON n.relationanchorpoint = h.childnodeanchor
-                WHERE n.nodeaggregateid = :nodeAggregateId
-                AND h.contentstreamid = :contentStreamId
+                WHERE h.contentstreamlayer = :contentStreamLayer
+                AND n.nodeaggregateid = :nodeAggregateId
                 AND h.dimensionspacepointhash = :dimensionSpacePointHash',
             [
-                'contentStreamId' => $contentStreamId->value,
+                'contentStreamLayer' => $contentStreamLayer->value,
                 'dimensionSpacePointHash' => $dimensionSpacePoint->hash,
                 'nodeAggregateId' => $nodeAggregateId->value
             ]
         )->fetchAssociative();
         if ($nodeRecord === false) {
-            throw new \InvalidArgumentException(sprintf('Failed to find hierarchy relation for content stream "%s", dimension space point "%s" and node aggregate id "%s"', $contentStreamId->value, $dimensionSpacePoint->hash, $nodeAggregateId->value), 1708617712);
+            throw new \InvalidArgumentException(sprintf('Failed to find hierarchy relation for content stream "%s", dimension space point "%s" and node aggregate id "%s"', $contentStreamLayer->value, $dimensionSpacePoint->hash, $nodeAggregateId->value), 1708617712);
         }
 
         return $nodeRecord;
@@ -327,4 +348,38 @@ private function transformPayloadTableToDataset(TableNode $payloadTable): array
 
         return $result;
     }
+
+    private function requireSingleWriteLayer(ContentStreamId $contentStreamId): ContentStreamLayer
+    {
+        $contentStreamLayers = $this->contentStreamLayerFinder()->getContentStreamLayers($contentStreamId);
+
+        if (!$contentStreamLayers->getWriteLayer()->equals($contentStreamLayers->getRootLayer())) {
+            throw new \RuntimeException(sprintf('For testing a single write layer is currently only supported for modification. Got %s for content stream %s.', $contentStreamLayers->toDebugString(), $contentStreamId->value), 1776786186);
+        }
+
+        return $contentStreamLayers->getWriteLayer();
+    }
+
+    /**
+     * DBAL Adapter specific assertion. The Error message strongly varies from adapter to adapter. Thus, we extend
+     *
+     * {@see ProjectionIntegrityViolationDetectionTrait::iExpectIntegrityViolationDetectionResultErrorNumberNToHaveCodeX} here.
+     *
+     * @Then I expect integrity violation detection result error number :errorNumber to have code :expectedErrorCode and message:
+     * @param int $errorNumber
+     * @param int $expectedErrorCode
+     */
+    public function iExpectIntegrityViolationDetectionResultErrorNumberNToHaveCodeXAndDbalAdapterMessage(int $errorNumber, int $expectedErrorCode, PyStringNode $message): void
+    {
+        $this->iExpectIntegrityViolationDetectionResultErrorNumberNToHaveCodeX($errorNumber, $expectedErrorCode);
+
+        /** @var \Neos\Error\Messages\Error $error */
+        $error = $this->lastIntegrityViolationDetectionResult->getErrors()[$errorNumber-1];
+
+        Assert::assertSame(
+            $message->getRaw(),
+            $error->render(),
+            "[{$error->getCode()}] " . $error->render()
+        );
+    }
 }

Neos.ContentGraph.DoctrineDbalAdapter/Tests/Behavior/Features/Bootstrap/DoctrineDbalProjectionSpyTrait.php

@@ -0,0 +1,25 @@
+<?php
+
+use PHPUnit\Framework\Assert;
+
+/**
+ * @internal custom non api inspections for the Doctrine DBAL content graph adapter to make assertions
+ */
+trait DoctrineDbalProjectionSpyTrait
+{
+    /**
+     * @Then I expect :number hierarchies to exist in the active write layer
+     */
+    public function iExpectNumberHierarchiesToExistInTheActiveWriteLayer(int $number)
+    {
+        $contentGraph = $this->currentContentRepository->getContentGraph($this->currentWorkspaceName);
+
+        $layers = $this->contentStreamLayerFinder()->getContentStreamLayers($contentGraph->getContentStreamId());
+
+        $count = $this->dbal->fetchOne(<<<SQL
+        SELECT COUNT(*) FROM {$this->tableNames()->hierarchyRelation()} WHERE contentStreamLayer = :contentStreamLayer
+        SQL, ['contentStreamLayer' => $layers->getWriteLayer()->value]);
+
+        Assert::assertEquals($number, $count);
+    }
+}

Neos.ContentGraph.DoctrineDbalAdapter/Tests/Behavior/Features/Bootstrap/FeatureContext.php

@@ -13,12 +13,16 @@
 
 use Behat\Behat\Context\Context as BehatContext;
 use Behat\Behat\Hook\Scope\BeforeScenarioScope;
+use Doctrine\DBAL\Connection;
 use Neos\Behat\FlowBootstrapTrait;
+use Neos\ContentGraph\DoctrineDbalAdapter\ContentGraphTableNames;
+use Neos\ContentGraph\DoctrineDbalAdapter\Domain\Repository\ContentStreamLayerFinder;
 use Neos\ContentRepository\Core\ContentRepository;
 use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceFactoryInterface;
 use Neos\ContentRepository\Core\Factory\ContentRepositoryServiceInterface;
 use Neos\ContentRepository\Core\SharedModel\ContentRepository\ContentRepositoryId;
 use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRBehavioralTestsSubjectProvider;
+use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRTestSuiteRuntimeVariables;
 use Neos\ContentRepository\TestSuite\Behavior\Features\Bootstrap\CRTestSuiteTrait;
 use Neos\ContentRepository\TestSuite\Fakes\FakeContentDimensionSourceFactory;
 use Neos\ContentRepository\TestSuite\Fakes\FakeNodeTypeManagerFactory;
@@ -31,17 +35,35 @@ class FeatureContext implements BehatContext
 {
     use FlowBootstrapTrait;
     use DoctrineDbalProjectionIntegrityViolatorTrait;
+    use DoctrineDbalProjectionSpyTrait;
     use CRTestSuiteTrait;
+    use CRTestSuiteRuntimeVariables;
     use CRBehavioralTestsSubjectProvider;
 
     protected ContentRepositoryRegistry $contentRepositoryRegistry;
 
+    protected Connection $dbal;
+
     public function __construct()
     {
         self::bootstrapFlow();
         $this->contentRepositoryRegistry = $this->getObject(ContentRepositoryRegistry::class);
+        $this->dbal = $this->getObject(Connection::class);
+    }
+
+    private function tableNames(): ContentGraphTableNames
+    {
+        return ContentGraphTableNames::create(
+            $this->currentContentRepository->id
+        );
+    }
 
-        $this->setupDbalGraphAdapterIntegrityViolationTrait();
+    private function contentStreamLayerFinder(): ContentStreamLayerFinder
+    {
+        return new ContentStreamLayerFinder(
+            $this->dbal,
+            $this->tableNames()
+        );
     }
 
     /**

Neos.ContentGraph.DoctrineDbalAdapter/Tests/Behavior/Features/Projection/ProjectionIntegrityViolationDetection/ContentStreamLayersIntegrityIsProvided.feature

@@ -0,0 +1,52 @@
+@contentrepository
+Feature: Run integrity violation detection regarding hierarchy relations and nodes
+
+  As a user of the CR I want to know whether there are nodes or hierarchy relations with invalid hashes or parents / children
+
+  Background:
+    Given using the following content dimensions:
+      | Identifier | Values      | Generalizations |
+      | language   | de, gsw, fr | gsw->de         |
+    And using the following node types:
+    """yaml
+    'Neos.ContentRepository.Testing:Document': []
+    """
+    And using identifier "default", I define a content repository
+    And I am in content repository "default"
+    And the command CreateRootWorkspace is executed with payload:
+      | Key                  | Value                |
+      | workspaceName        | "live"               |
+      | newContentStreamId   | "cs-identifier"      |
+    And I am in workspace "live" and dimension space point {}
+    And the command CreateRootNodeAggregateWithNode is executed with payload:
+      | Key             | Value                         |
+      | nodeAggregateId | "lady-eleonode-rootford"      |
+      | nodeTypeName    | "Neos.ContentRepository:Root" |
+
+  Scenario: Detach a hierarchy relation from its parent
+    When the command CreateWorkspace is executed with payload:
+      | Key                | Value                |
+      | workspaceName      | "user-test"          |
+      | baseWorkspaceName  | "live"               |
+      | newContentStreamId | "user-cs-identifier" |
+
+    When the command CreateNodeAggregateWithNode is executed with payload:
+      | Key                       | Value                                     |
+      | workspaceName             | "live"                                    |
+      | originDimensionSpacePoint | {"language":"de"}                         |
+      | nodeAggregateId           | "sir-david-nodenborough"                  |
+      | nodeTypeName              | "Neos.ContentRepository.Testing:Document" |
+      | parentNodeAggregateId     | "lady-eleonode-rootford"                  |
+
+    # Basically a DeleteWorkspace without the event ContentStreamWasRemoved as we avoid the internal cleanup
+    When the content stream "user-cs-identifier" was removed without layer cleanup
+    When the event WorkspaceWasRemoved was published with payload:
+      | Key           | Value       |
+      | workspaceName | "user-test" |
+
+    And I run integrity violation detection
+    Then I expect the integrity violation detection result to contain exactly 1 error
+    And I expect integrity violation detection result error number 1 to have code 1597909228 and message:
+    """
+    Redundant layer 1 to 2 found for content streams cs-identifier
+    """