Integration tests/ fix failed tests

.github/workflows/full-integration-tests.yaml

@@ -1,7 +1,12 @@
 name: Integration Tests - Full Test Suite
 run-name: ${{ github.actor }} is running Full Integration Test Suite
 on:
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      test_directory:
+        description: 'Test directory to run integration tests'
+        type: string
+        required: false
 
 jobs:
   call-workflow:
@@ -11,3 +16,4 @@ jobs:
     with:
       repository: ${{ github.repository }}
       head: ${{ github.sha }}
+      test_directory: ${{ github.event.inputs.test_directory }}

app/code/Magento/Catalog/Model/Indexer/Category/Product/AbstractAction.php

@@ -187,7 +187,7 @@ abstract public function execute();
      */
     protected function reindex()
     {
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             if ($this->getPathFromCategoryId($store->getRootCategoryId())) {
                 $this->setCurrentStore($store);
                 $this->reindexRootCategory($store);
@@ -286,7 +286,7 @@ protected function getPathFromCategoryId($categoryId)
                     ['path']
                 )->where(
                     'entity_id = ?',
-                    $categoryId == 0 ? 1 : $categoryId
+                    $categoryId
                 )
             );
         }
@@ -323,6 +323,10 @@ protected function getNonAnchorCategoriesSelect(Store $store)
                 ['ccp' => $this->getTable('catalog_category_product')],
                 'ccp.category_id = cc.entity_id',
                 []
+            )->joinInner(
+                ['cpw' => $this->getTable('catalog_product_website')],
+                'cpw.product_id = ccp.product_id',
+                []
             )->joinInner(
                 ['cpe' => $this->getTable('catalog_product_entity')],
                 'ccp.product_id = cpe.entity_id',
@@ -351,6 +355,11 @@ protected function getNonAnchorCategoriesSelect(Store $store)
                 ' AND cpvs.store_id = ' .
                 $store->getId(),
                 []
+            )->where(
+                'cc.path LIKE ' . $this->connection->quote($rootPath . '/%')
+            )->where(
+                'cpw.website_id = ?',
+                $store->getWebsiteId()
             )->where(
                 $this->connection->getIfNullSql('cpss.value', 'cpsd.value') . ' = ?',
                 \Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_ENABLED
@@ -374,19 +383,6 @@ protected function getNonAnchorCategoriesSelect(Store $store)
                 ]
             );
 
-            if ($store->getId() != 0) {
-                $select->joinInner(
-                    ['cpw' => $this->getTable('catalog_product_website')],
-                    'cpw.product_id = ccp.product_id',
-                    []
-                )->where(
-                    'cpw.website_id = ?',
-                    $store->getWebsiteId()
-                )->where(
-                    'cc.path LIKE ' . $this->connection->quote($rootPath . '/%')
-                );
-            }
-
             $this->addFilteringByChildProductsToSelect($select, $store);
 
             $this->nonAnchorSelects[$store->getId()] = $select;
@@ -537,7 +533,6 @@ protected function createAnchorSelect(Store $store)
         $visibilityAttributeId = $this->config->getAttribute(Product::ENTITY, 'visibility')->getId();
         $rootCatIds = explode('/', $this->getPathFromCategoryId($store->getRootCategoryId()));
         array_pop($rootCatIds);
-        $rootCatIds = $rootCatIds ?: [1];
 
         $temporaryTreeTable = $this->makeTempCategoryTreeIndex();
 
@@ -569,6 +564,10 @@ protected function createAnchorSelect(Store $store)
             ['cpe' => $this->getTable('catalog_product_entity')],
             'ccp.product_id = cpe.entity_id',
             []
+        )->joinInner(
+            ['cpw' => $this->getTable('catalog_product_website')],
+            'cpw.product_id = ccp.product_id',
+            []
         )->joinInner(
             ['cpsd' => $this->getTable('catalog_product_entity_int')],
             'cpsd.' . $productLinkField . ' = cpe.' . $productLinkField . ' AND cpsd.store_id = 0'
@@ -603,6 +602,9 @@ protected function createAnchorSelect(Store $store)
             . ' AND ccas.attribute_id = ccad.attribute_id AND ccas.store_id = ' .
             $store->getId(),
             []
+        )->where(
+            'cpw.website_id = ?',
+            $store->getWebsiteId()
         )->where(
             $this->connection->getIfNullSql('cpss.value', 'cpsd.value') . ' = ?',
             \Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_ENABLED
@@ -629,17 +631,6 @@ protected function createAnchorSelect(Store $store)
             ]
         );
 
-        if ($store->getId() != 0) {
-            $select->joinInner(
-                ['cpw' => $this->getTable('catalog_product_website')],
-                'cpw.product_id = ccp.product_id',
-                []
-            )->where(
-                'cpw.website_id = ?',
-                $store->getWebsiteId()
-            );
-        }
-
         $this->addFilteringByChildProductsToSelect($select, $store);
 
         return $select;
@@ -827,6 +818,10 @@ protected function getAllProducts(Store $store)
             $select = $this->connection->select()->from(
                 ['cp' => $this->getTable('catalog_product_entity')],
                 []
+            )->joinInner(
+                ['cpw' => $this->getTable('catalog_product_website')],
+                'cpw.product_id = cp.entity_id',
+                []
             )->joinInner(
                 ['cpsd' => $this->getTable('catalog_product_entity_int')],
                 'cpsd.' . $linkField . ' = cp.' . $linkField . ' AND cpsd.store_id = 0' .
@@ -855,6 +850,9 @@ protected function getAllProducts(Store $store)
                 ['ccp' => $this->getTable('catalog_category_product')],
                 'ccp.product_id = cp.entity_id',
                 []
+            )->where(
+                'cpw.website_id = ?',
+                $store->getWebsiteId()
             )->where(
                 $this->connection->getIfNullSql('cpss.value', 'cpsd.value') . ' = ?',
                 \Magento\Catalog\Model\Product\Attribute\Source\Status::STATUS_ENABLED
@@ -884,17 +882,6 @@ protected function getAllProducts(Store $store)
                 ]
             );
 
-            if ($store->getId() != 0) {
-                $select->joinInner(
-                    ['cpw' => $this->getTable('catalog_product_website')],
-                    'cpw.product_id = ccp.product_id',
-                    []
-                )->where(
-                    'cpw.website_id = ?',
-                    $store->getWebsiteId()
-                );
-            }
-
             $this->productsSelects[$store->getId()] = $select;
         }
 

app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Full.php

@@ -133,7 +133,7 @@ public function __construct(
      */
     private function createTables(): void
     {
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             $this->tableMaintainer->createTablesForStore((int)$store->getId());
         }
     }
@@ -145,7 +145,7 @@ private function createTables(): void
      */
     private function clearReplicaTables(): void
     {
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             $this->connection->truncateTable($this->tableMaintainer->getMainReplicaTable((int)$store->getId()));
         }
     }
@@ -158,7 +158,7 @@ private function clearReplicaTables(): void
     private function switchTables(): void
     {
         $tablesToSwitch = [];
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             $tablesToSwitch[] = $this->tableMaintainer->getMainTable((int)$store->getId());
         }
         $this->activeTableSwitcher->switchTable($this->connection, $tablesToSwitch);
@@ -188,7 +188,7 @@ protected function reindex(): void
     {
         $userFunctions = [];
 
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             if ($this->getPathFromCategoryId($store->getRootCategoryId())) {
                 $userFunctions[$store->getId()] = function () use ($store) {
                     $this->reindexStore($store);

app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Rows.php

@@ -117,7 +117,7 @@ public function execute(array $entityIds = [], $useTempTable = false)
             || ($indexer->isScheduled() && !$useTempTable)
             || ($indexer->isScheduled() && $useTempTable && !$workingState)) {
             if ($useTempTable && !$workingState && $indexer->isScheduled()) {
-                foreach ($this->storeManager->getStores(true) as $store) {
+                foreach ($this->storeManager->getStores() as $store) {
                     $this->connection->truncateTable($this->getIndexTable($store->getId()));
                 }
             } else {
@@ -130,7 +130,7 @@ public function execute(array $entityIds = [], $useTempTable = false)
             $workingState = $this->isWorkingState();
 
             if ($useTempTable && !$workingState && $indexer->isScheduled()) {
-                foreach ($this->storeManager->getStores(true) as $store) {
+                foreach ($this->storeManager->getStores() as $store) {
                     $removalCategoryIds = array_diff($this->limitationByCategories, [$this->getRootCategoryId($store)]);
                     $this->connection->delete(
                         $this->tableMaintainer->getMainTable($store->getId()),
@@ -204,7 +204,7 @@ private function getRootCategoryId($store)
      */
     private function removeEntries()
     {
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             $removalCategoryIds = array_diff($this->limitationByCategories, [$this->getRootCategoryId($store)]);
             $this->connection->delete(
                 $this->getIndexTable($store->getId()),

app/code/Magento/Catalog/Model/Indexer/Category/Product/Plugin/StoreGroup.php

@@ -78,7 +78,7 @@ protected function validate(AbstractModel $group)
      */
     public function afterDelete(AbstractDb $subject, AbstractDb $objectResource, AbstractModel $storeGroup)
     {
-        foreach ($storeGroup->getStores(true) as $store) {
+        foreach ($storeGroup->getStores() as $store) {
             $this->tableMaintainer->dropTablesForStore((int)$store->getId());
         }
         return $objectResource;

app/code/Magento/Catalog/Model/Indexer/Product/Category/Action/Rows.php

@@ -115,7 +115,7 @@ public function execute(array $entityIds = [], $useTempTable = false)
             $affectedCategories = $this->getCategoryIdsFromIndex($idsToBeReIndexed);
 
             if ($useTempTable && !$workingState && $indexer->isScheduled()) {
-                foreach ($this->storeManager->getStores(true) as $store) {
+                foreach ($this->storeManager->getStores() as $store) {
                     $this->connection->truncateTable($this->getIndexTable($store->getId()));
                 }
             } else {
@@ -127,7 +127,7 @@ public function execute(array $entityIds = [], $useTempTable = false)
             $workingState = $this->isWorkingState();
 
             if ($useTempTable && !$workingState && $indexer->isScheduled()) {
-                foreach ($this->storeManager->getStores(true) as $store) {
+                foreach ($this->storeManager->getStores() as $store) {
                     $this->connection->delete(
                         $this->tableMaintainer->getMainTable($store->getId()),
                         ['product_id IN (?)' => $this->limitationByProducts]
@@ -236,7 +236,7 @@ private function registerCategories(array $categoryIds)
      */
     protected function removeEntries()
     {
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             $this->connection->delete(
                 $this->getIndexTable($store->getId()),
                 ['product_id IN (?)' => $this->limitationByProducts]
@@ -299,7 +299,7 @@ protected function isRangingNeeded()
     private function getCategoryIdsFromIndex(array $productIds): array
     {
         $categoryIds = [];
-        foreach ($this->storeManager->getStores(true) as $store) {
+        foreach ($this->storeManager->getStores() as $store) {
             $storeCategories = $this->connection->fetchCol(
                 $this->connection->select()
                     ->from($this->getIndexTable($store->getId()), ['category_id'])

app/code/Magento/Catalog/Model/ResourceModel/Category/Collection.php

@@ -11,7 +11,6 @@
 use Magento\Framework\App\Config\ScopeConfigInterface;
 use Magento\Framework\DB\Select;
 use Magento\Store\Model\ScopeInterface;
-use Magento\Catalog\Model\Indexer\Category\Product\TableMaintainer;
 
 /**
  * Category resource collection
@@ -76,11 +75,6 @@ class Collection extends \Magento\Catalog\Model\ResourceModel\Collection\Abstrac
      */
     private $catalogProductVisibility;
 
-    /**
-     * @var TableMaintainer
-     */
-    private $tableMaintainer;
-
     /**
      * Constructor
      * @param \Magento\Framework\Data\Collection\EntityFactory $entityFactory
@@ -130,7 +124,6 @@ public function __construct(
             \Magento\Framework\App\ObjectManager::getInstance()->get(ScopeConfigInterface::class);
         $this->catalogProductVisibility = $catalogProductVisibility ?:
             \Magento\Framework\App\ObjectManager::getInstance()->get(Visibility::class);
-        $this->tableMaintainer = \Magento\Framework\App\ObjectManager::getInstance()->get(TableMaintainer::class);
     }
 
     /**
@@ -347,7 +340,7 @@ public function loadProductCount($items, $countRegular = true, $countAnchor = tr
             foreach ($anchor as $item) {
                 $productsCount = isset($categoryProductsCount[$item->getId()])
                     ? (int)$categoryProductsCount[$item->getId()]
-                    : 0;
+                    : $this->getProductsCountFromCategoryTable($item, $websiteId);
                 $item->setProductCount($productsCount);
             }
         }
@@ -526,6 +519,45 @@ public function getProductTable()
         return $this->_productTable;
     }
 
+    /**
+     * Get products count using catalog_category_entity table
+     *
+     * @param Category $item
+     * @param string $websiteId
+     * @return int
+     */
+    private function getProductsCountFromCategoryTable(Category $item, string $websiteId): int
+    {
+        $productCount = 0;
+
+        if ($item->getAllChildren()) {
+            $bind = ['entity_id' => $item->getId(), 'c_path' => $item->getPath() . '/%'];
+            $select = $this->_conn->select();
+            $select->from(
+                ['main_table' => $this->getProductTable()],
+                new \Zend_Db_Expr('COUNT(DISTINCT main_table.product_id)')
+            )->joinInner(
+                ['e' => $this->getTable('catalog_category_entity')],
+                'main_table.category_id=e.entity_id',
+                []
+            )->where(
+                '(e.entity_id = :entity_id OR e.path LIKE :c_path)'
+            );
+            if ($websiteId) {
+                $select->join(
+                    ['w' => $this->getProductWebsiteTable()],
+                    'main_table.product_id = w.product_id',
+                    []
+                )->where(
+                    'w.website_id = ?',
+                    $websiteId
+                );
+            }
+            $productCount = (int)$this->_conn->fetchOne($select, $bind);
+        }
+        return $productCount;
+    }
+
     /**
      * Get query for retrieve count of products per category
      *
@@ -535,7 +567,7 @@ public function getProductTable()
      */
     private function getProductsCountQuery(array $categoryIds, $addVisibilityFilter = true): Select
     {
-        $categoryTable = $this->tableMaintainer->getMainTable($this->getProductStoreId());
+        $categoryTable = $this->_resource->getTableName('catalog_category_product_index');
         $select = $this->_conn->select()
             ->from(
                 ['cat_index' => $categoryTable],

app/code/Magento/Catalog/Setup/Patch/Schema/EnableSegmentation.php

@@ -40,7 +40,7 @@ public function apply()
         $this->schemaSetup->startSetup();
         $setup = $this->schemaSetup;
 
-        $storeSelect = $setup->getConnection()->select()->from($setup->getTable('store'));
+        $storeSelect = $setup->getConnection()->select()->from($setup->getTable('store'))->where('store_id > 0');
         foreach ($setup->getConnection()->fetchAll($storeSelect) as $store) {
             $indexTable = $setup->getTable('catalog_category_product_index') .
                 '_' .
@@ -54,7 +54,7 @@ public function apply()
                     )
                 );
             }
-            if ($store['store_id'] > 0 && !$setup->getConnection()->isTableExists($indexTable . '_replica')) {
+            if (!$setup->getConnection()->isTableExists($indexTable . '_replica')) {
                 $setup->getConnection()->createTable(
                     $setup->getConnection()->createTableByDdl(
                         $setup->getTable('catalog_category_product_index'),

app/code/Magento/CatalogInventory/Model/ResourceModel/Stock.php

@@ -162,7 +162,9 @@ public function lockProductsStock(array $productIds, $websiteId)
             $items[$si['product_id']] = $si;
         }
         foreach ($this->getConnection()->fetchAll($selectProducts) as $p) {
-            $items[$p['product_id']]['type_id'] = $p['type_id'];
+            if (isset($items[$p['product_id']])) {
+                $items[$p['product_id']]['type_id'] = $p['type_id'];
+            }
         }
 
         return $items;