Stream XLSX/XML interpreter rows to reduce memory usage

src/DataSource/DataLoader/Xlsx/PhpOfficeXlsxDataLoader.php

@@ -16,12 +16,12 @@ class PhpOfficeXlsxDataLoader implements XlsxDataLoaderInterface
 {
     /**
      * @param string $file
-     * 
+     *
      * @param string $sheet
-     * 
-     * @return array
+     *
+     * @return iterable<array>
      */
-    public function getRows(string $file, string $sheet): array{
+    public function getRows(string $file, string $sheet): iterable{
 
         $reader = IOFactory::createReaderForFile($file);
         $reader->setReadDataOnly(true);

src/DataSource/DataLoader/Xlsx/SpoutXlsxDataLoader.php

@@ -6,46 +6,45 @@
 use OpenSpout\Reader\XLSX\Reader;
 
 /**
- * PHPOffice uses significantly more memory but is faster. Use this implementation when you have lots of memory and want speed.
+ * Streams rows one by one so large files never have to be loaded into memory at once.
  */
 
 class SpoutXlsxDataLoader implements XlsxDataLoaderInterface
 {
     /**
      * @param string $file
-     * 
+     *
      * @param string $sheet
-     * 
-     * @return array
+     *
+     * @return iterable<array>
      */
-    public function getRows(string $file, string $sheet): array{
+    public function getRows(string $file, string $sheet): iterable{
 
-        $data=array();
         $reader = new Reader();
         $reader->open($file);
 
-        foreach($reader->getSheetIterator() as $currentSheet){
-            if($currentSheet->getName() != $sheet){
-                continue;
-            }
+        try {
+            foreach($reader->getSheetIterator() as $currentSheet){
+                if($currentSheet->getName() != $sheet){
+                    continue;
+                }
 
-            foreach($currentSheet->getRowIterator() as $row){
-                $cells = $row->getCells();
-                $dataRow = [];
-                foreach ($cells as $cell) {
-                    if($cell instanceof FormulaCell){
-                        $dataRow[] = $cell->getComputedValue();
-                    }else{
-                        $dataRow[] = $cell->getValue();
+                foreach($currentSheet->getRowIterator() as $row){
+                    $cells = $row->getCells();
+                    $dataRow = [];
+                    foreach ($cells as $cell) {
+                        if($cell instanceof FormulaCell){
+                            $dataRow[] = $cell->getComputedValue();
+                        }else{
+                            $dataRow[] = $cell->getValue();
+                        }
                     }
+                    yield $dataRow;
                 }
-                $data[]=$dataRow;
             }
+        } finally {
+            $reader->close();
         }
-
-        $reader->close();
-
-        return $data;
     }
-    
+
 }

src/DataSource/DataLoader/Xlsx/XlsxDataLoaderInterface.php

@@ -6,12 +6,16 @@
 interface XlsxDataLoaderInterface
 {
     /**
+     * Returns the rows of the given sheet one by one. Implementations should stream
+     * (yield) rows instead of building the full array in memory, so large files do
+     * not exhaust the memory limit.
+     *
      * @param string $file
-     * 
+     *
      * @param string $sheet
-     * 
-     * @return array
+     *
+     * @return iterable<array>
      */
-    public function getRows(string $file, string $sheet): array;
-    
-}
+    public function getRows(string $file, string $sheet): iterable;
+
+}

src/DataSource/Interpreter/AdvancedXlsxFileInterpreter.php

@@ -23,21 +23,22 @@ protected function doInterpretFileAndCallProcessRow(string $path): void
         $this->uniqueHashes = array();
 
         $excelLoader = XlsxDataLoaderFactory::getExcelDataLoader();
-        $data = $excelLoader->getRows($path, $this->sheetName);
 
-        // Header row is 1-indexed, array is 0-indexed
-        $headerRowIndex = $this->headerRow - 1;
-
-        // Get header row for column names
+        $expressionLanguage = strlen($this->rowFilter) > 0 ? new ExpressionLanguage() : null;
         $headerRow = null;
-        if ($this->saveHeaderName && isset($data[$headerRowIndex])) {
-            $headerRow = $data[$headerRowIndex];
-        }
+        $rowNumber = 0;
 
-        // Skip rows up to and including the header row
-        $data = array_slice($data, $this->headerRow);
+        // Rows are streamed one by one so large files never have to be loaded into memory at once.
+        foreach ($excelLoader->getRows($path, $this->sheetName) as $rowData) {
+            $rowNumber++;
 
-        foreach ($data as $rowData) {
+            // Rows up to and including the header row (1-indexed) are skipped
+            if ($rowNumber <= $this->headerRow) {
+                if ($rowNumber === $this->headerRow && $this->saveHeaderName) {
+                    $headerRow = $rowData;
+                }
+                continue;
+            }
 
             $hashKey = '';
 
@@ -49,9 +50,7 @@ protected function doInterpretFileAndCallProcessRow(string $path): void
                 continue;
             }
 
-            if(strlen($this->rowFilter) > 0){
-                $expressionLanguage = new ExpressionLanguage();
-
+            if($expressionLanguage !== null){
                 $filterResult = $expressionLanguage->evaluate($this->rowFilter, ['row' => $rowData]);
 
                 if(!$filterResult){

src/DataSource/Interpreter/BulkXlsxFileInterpreter.php

@@ -30,19 +30,6 @@ protected function doInterpretFileAndCallProcessRow(string $path): void
         $this->uniqueHashes = array();
 
         $excelLoader = XlsxDataLoaderFactory::getExcelDataLoader();
-        $data = $excelLoader->getRows($path, $this->sheetName);
-
-        // Header row is 1-indexed, array is 0-indexed
-        $headerRowIndex = $this->headerRow - 1;
-
-        // Get header row for column names
-        $headerRow = null;
-        if ($this->saveHeaderName && isset($data[$headerRowIndex])) {
-            $headerRow = $data[$headerRowIndex];
-        }
-
-        // Skip rows up to and including the header row
-        $data = array_slice($data, $this->headerRow);
 
         $tmpCsv = tempnam(sys_get_temp_dir(), 'pimcore_bulk_load');
         $options = new Options();
@@ -53,7 +40,21 @@ protected function doInterpretFileAndCallProcessRow(string $path): void
         $carbonNow = Carbon::now();
         $db = Db::get();
 
-        foreach ($data as $rowData) {
+        $expressionLanguage = strlen($this->rowFilter) > 0 ? new ExpressionLanguage() : null;
+        $headerRow = null;
+        $rowNumber = 0;
+
+        // Rows are streamed one by one so large files never have to be loaded into memory at once.
+        foreach ($excelLoader->getRows($path, $this->sheetName) as $rowData) {
+            $rowNumber++;
+
+            // Rows up to and including the header row (1-indexed) are skipped
+            if ($rowNumber <= $this->headerRow) {
+                if ($rowNumber === $this->headerRow && $this->saveHeaderName) {
+                    $headerRow = $rowData;
+                }
+                continue;
+            }
 
             $hashKey = '';
 
@@ -65,9 +66,7 @@ protected function doInterpretFileAndCallProcessRow(string $path): void
                 continue;
             }
 
-            if(strlen($this->rowFilter) > 0){
-                $expressionLanguage = new ExpressionLanguage();
-
+            if($expressionLanguage !== null){
                 $filterResult = $expressionLanguage->evaluate($this->rowFilter, ['row' => $rowData]);
 
                 if(!$filterResult){

src/DataSource/Interpreter/ChunkedRowsReadFilter.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace TorqIT\DataImporterExtensionsBundle\DataSource\Interpreter;
+
+use PhpOffice\PhpSpreadsheet\Reader\IReadFilter;
+
+/**
+ * Restricts spreadsheet reading to a contiguous row range so imports can be
+ * processed chunk by chunk instead of loading the whole workbook into memory.
+ */
+class ChunkedRowsReadFilter implements IReadFilter
+{
+    public function __construct(
+        private readonly int $startRow,
+        private readonly int $endRow,
+    ) {
+    }
+
+    public function readCell(string $columnAddress, int $row, string $worksheetName = ''): bool
+    {
+        return $row >= $this->startRow && $row <= $this->endRow;
+    }
+}

src/Override/CustomXlsxFileInterpreter.php

@@ -9,31 +9,53 @@
 use PhpOffice\PhpSpreadsheet\RichText\RichText;
 use PhpOffice\PhpSpreadsheet\Worksheet\Worksheet;
 use Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter\AbstractInterpreter;
+use Pimcore\Bundle\DataImporterBundle\Exception\InvalidConfigurationException;
 use Pimcore\Bundle\DataImporterBundle\Preview\Model\PreviewData;
+use TorqIT\DataImporterExtensionsBundle\DataSource\Interpreter\ChunkedRowsReadFilter;
 use TorqIT\DataImporterExtensionsBundle\DataSource\Interpreter\PreviewRowsReadFilter;
 
 // Copy of Pimcore\Bundle\DataImporterBundle\DataSource\Interpreter\XlsxFileInterpreter
 class CustomXlsxFileInterpreter extends AbstractInterpreter
 {
+    protected const IMPORT_CHUNK_SIZE = 1000;
+
     protected bool $skipFirstRow;
     protected string $sheetName;
 
+    /**
+     * Reads the workbook in bounded chunks instead of one toArray() call -
+     * large files would otherwise exhaust the memory limit.
+     */
     protected function doInterpretFileAndCallProcessRow(string $path): void
     {
-        $reader = IOFactory::createReaderForFile($path);
-        $reader->setReadDataOnly(true);
-        $spreadSheet = $reader->load($path);
+        $worksheetInfo = $this->getWorksheetInfo($path);
 
-        $spreadSheet->setActiveSheetIndexByName($this->sheetName);
+        if ($worksheetInfo === null) {
+            throw new InvalidConfigurationException(sprintf('Sheet "%s" not found in file.', $this->sheetName));
+        }
 
-        $data = $spreadSheet->getActiveSheet()->toArray();
+        $totalRows = $worksheetInfo['totalRows'];
+        $lastColumnLetter = $worksheetInfo['lastColumnLetter'];
+        $startRow = $this->skipFirstRow ? 2 : 1;
 
-        if ($this->skipFirstRow) {
-            array_shift($data);
-        }
+        for ($chunkStart = $startRow; $chunkStart <= $totalRows; $chunkStart += static::IMPORT_CHUNK_SIZE) {
+            $chunkEnd = min($chunkStart + static::IMPORT_CHUNK_SIZE - 1, $totalRows);
+
+            $reader = IOFactory::createReaderForFile($path);
+            $reader->setReadDataOnly(true);
+            $reader->setLoadSheetsOnly($this->sheetName);
+            $reader->setReadFilter(new ChunkedRowsReadFilter($chunkStart, $chunkEnd));
+            $spreadSheet = $reader->load($path);
+
+            $spreadSheet->setActiveSheetIndexByName($this->sheetName);
+            $sheet = $spreadSheet->getActiveSheet();
 
-        foreach ($data as $rowData) {
-            $this->processImportRow($rowData);
+            for ($rowNumber = $chunkStart; $rowNumber <= $chunkEnd; $rowNumber++) {
+                $this->processImportRow($this->readPreviewRow($sheet, $rowNumber, $lastColumnLetter));
+            }
+
+            $spreadSheet->disconnectWorksheets();
+            unset($spreadSheet);
         }
     }
 
@@ -139,16 +161,30 @@ protected function getPreviewCellValue(Cell $cell): mixed
     }
 
     protected function getTotalRows(string $path): int
+    {
+        return (int)($this->getWorksheetInfo($path)['totalRows'] ?? 0);
+    }
+
+    /**
+     * Reads the worksheet dimensions without loading any cells.
+     * Returns null when the configured sheet does not exist in the file.
+     *
+     * @return array{totalRows: int, lastColumnLetter: string}|null
+     */
+    protected function getWorksheetInfo(string $path): ?array
     {
         $reader = IOFactory::createReaderForFile($path);
 
         foreach ($reader->listWorksheetInfo($path) as $worksheetInfo) {
             if (($worksheetInfo['worksheetName'] ?? null) === $this->sheetName) {
-                return (int)($worksheetInfo['totalRows'] ?? 0);
+                return [
+                    'totalRows' => (int)($worksheetInfo['totalRows'] ?? 0),
+                    'lastColumnLetter' => (string)($worksheetInfo['lastColumnLetter'] ?? 'A'),
+                ];
             }
         }
 
-        return 0;
+        return null;
     }
 
     public function setSettings(array $settings): void