Skip to content

Commit d022bff

Browse files
sangnguyenplussangnguyenplus
committed
Improve export performance
1 parent 3f5b8df commit d022bff

File tree

9 files changed

+515
-15
lines changed

9 files changed

+515
-15
lines changed

‎config/data-synchronize.php

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,11 @@
1717
'disk' => 'local',
1818
'path' => 'data-synchronize',
1919
],
20+
'export' => [
21+
'chunk_size' => env('EXPORT_CHUNK_SIZE', 400),
22+
'memory_limit' => env('EXPORT_MEMORY_LIMIT', '512M'),
23+
'time_limit' => env('EXPORT_TIME_LIMIT', 0),
24+
'optimize_memory' => env('EXPORT_OPTIMIZE_MEMORY', true),
25+
'use_chunked' => env('EXPORT_USE_CHUNKED', true),
26+
],
2027
];

‎resources/lang/en/data-synchronize.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
'export' => [
4949
'name' => 'Export',
5050
'heading' => 'Export :label',
51+
'excel_not_supported_for_large_exports' => 'Excel format is not supported for large exports (:count items). Please use CSV format instead for better performance and reliability.',
5152

5253
'form' => [
5354
'all_columns_disabled' => 'Following columns will be exported: :columns.',

‎resources/views/export.blade.php

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -102,18 +102,15 @@ class="export-column"
102102
<script>
103103
document.addEventListener('DOMContentLoaded', function() {
104104
const form = document.querySelector('.data-synchronize-export-form');
105-
const storageKey = 'data-synchronize-export-form-' + window.location.pathname;
105+
const storageKey = 'data-synchronize-export-form-v-1' + window.location.pathname;
106106
const columnCheckboxes = form.querySelectorAll('.export-column');
107107
const checkAllButton = form.querySelector('.check-all-columns');
108108
109-
// Function to save form values to localStorage
110109
function saveFormValues() {
111110
const formData = new FormData(form);
112111
const values = {};
113112
114-
// Save all form fields except _token
115113
for (const [key, value] of formData.entries()) {
116-
// Skip the CSRF token field
117114
if (key === '_token') {
118115
continue;
119116
}
@@ -131,27 +128,22 @@ function saveFormValues() {
131128
localStorage.setItem(storageKey, JSON.stringify(values));
132129
}
133130
134-
// Function to restore form values from localStorage
135131
function restoreFormValues() {
136132
const savedValues = localStorage.getItem(storageKey);
137133
if (!savedValues) return;
138134
139135
const values = JSON.parse(savedValues);
140136
141-
// Restore all form fields except _token
142137
Object.entries(values).forEach(([key, value]) => {
143-
// Skip the CSRF token field
144138
if (key === '_token') {
145139
return;
146140
}
147141
148142
if (key === 'columns') {
149-
// Handle checkboxes
150143
columnCheckboxes.forEach(checkbox => {
151144
checkbox.checked = value.includes(checkbox.value);
152145
});
153146
} else {
154-
// Handle other form fields
155147
const input = form.querySelector(`[name="${key}"]`);
156148
if (input) {
157149
if (input.type === 'radio') {
@@ -167,7 +159,6 @@ function restoreFormValues() {
167159
});
168160
}
169161
170-
// Handle check all functionality
171162
checkAllButton.addEventListener('click', function(e) {
172163
e.preventDefault();
173164
const allChecked = Array.from(columnCheckboxes).every(checkbox => checkbox.checked);
@@ -181,11 +172,9 @@ function restoreFormValues() {
181172
saveFormValues();
182173
});
183174
184-
// Save form values when any input changes
185175
form.addEventListener('change', saveFormValues);
186176
form.addEventListener('input', saveFormValues);
187177
188-
// Restore form values when page loads
189178
restoreFormValues();
190179
});
191180
</script>

‎src/Commands/ExportCommand.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,25 @@ public function handle(): void
5757

5858
$exporter->format($format);
5959

60+
// Configure memory optimization
61+
if ($this->option('optimize-memory')) {
62+
$exporter->setOptimizeMemory(true);
63+
}
64+
65+
// Configure chunk size if provided
66+
if ($chunkSize = $this->option('chunk-size')) {
67+
if (method_exists($exporter, 'setChunkSize')) {
68+
$exporter->setChunkSize((int) $chunkSize);
69+
}
70+
}
71+
72+
// Show export info
6073
$this->components->info("Exporting {$exporter->getLabel()} to <comment>{$path}</comment>");
6174

75+
if (method_exists($exporter, 'getChunkSize')) {
76+
$this->components->info("Using chunk size: {$exporter->getChunkSize()}");
77+
}
78+
6279
try {
6380
$exporter->export()
6481
->getFile()
@@ -80,6 +97,8 @@ protected function getOptions(): array
8097
{
8198
return [
8299
['format', null, InputOption::VALUE_OPTIONAL, 'The format of the file (csv, xls, xlsx)'],
100+
['chunk-size', null, InputOption::VALUE_OPTIONAL, 'Number of records to process per chunk'],
101+
['optimize-memory', null, InputOption::VALUE_NONE, 'Enable memory optimization for large exports'],
83102
];
84103
}
85104

‎src/Commands/TestLargeExportCommand.php

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
<?php
2+
3+
namespace Botble\DataSynchronize\Commands;
4+
5+
use Botble\Ecommerce\Exporters\ProductExporter;
6+
use Illuminate\Console\Command;
7+
use Symfony\Component\Console\Attribute\AsCommand;
8+
9+
#[AsCommand(name: 'data-synchronize:test-large-export', description: 'Test large product export functionality')]
10+
class TestLargeExportCommand extends Command
11+
{
12+
public function handle(): void
13+
{
14+
$this->info('Testing Large Product Export...');
15+
16+
try {
17+
$exporter = new ProductExporter();
18+
19+
// Get counts
20+
$counters = $exporter->getCounters();
21+
$this->info('Export Statistics:');
22+
foreach ($counters as $counter) {
23+
$this->line("- {$counter->getLabel()}: {$counter->getValue()}");
24+
}
25+
26+
// Configure for large export
27+
$exporter->setChunkSize(200)
28+
->useChunkedExport(true)
29+
->setOptimizeMemory(true)
30+
->enableStreamingMode(true)
31+
->setIncludeVariations(true)
32+
->format('csv');
33+
34+
$this->info("\nExport Configuration:");
35+
$this->line("- Chunk Size: {$exporter->getChunkSize()}");
36+
$this->line('- Streaming Mode: ' . ($exporter->isStreamingMode() ? 'Enabled' : 'Disabled'));
37+
$this->line('- Include Variations: Yes');
38+
$this->line('- Format: CSV');
39+
40+
// Test query
41+
$this->info("\nTesting export query...");
42+
$query = $exporter->query();
43+
$count = $query->count();
44+
$this->line("- Query will process: {$count} main products");
45+
46+
// Test chunk processing
47+
$this->info("\nTesting chunk processing...");
48+
$processed = 0;
49+
$rows = 0;
50+
51+
$bar = $this->output->createProgressBar($count);
52+
$bar->start();
53+
54+
$query->chunk($exporter->chunkSize(), function ($products) use (&$processed, &$rows, $bar, $exporter) {
55+
foreach ($products as $product) {
56+
$processed++;
57+
$rows++; // Main product
58+
59+
if ($product->variations->count() > 0) {
60+
$rows += $product->variations->count(); // Variations
61+
}
62+
63+
$bar->advance();
64+
65+
if ($processed >= 10) {
66+
// Just test first 10 products
67+
return false;
68+
}
69+
}
70+
});
71+
72+
$bar->finish();
73+
$this->newLine();
74+
75+
$this->info("\nTest Results:");
76+
$this->line("- Processed {$processed} main products");
77+
$this->line("- Would generate {$rows} total rows");
78+
$this->line('- Memory usage: ' . round(memory_get_usage() / 1024 / 1024, 2) . ' MB');
79+
80+
$this->info("\nStreaming export is properly configured and ready to handle large datasets!");
81+
82+
} catch (\Exception $e) {
83+
$this->error('Error: ' . $e->getMessage());
84+
$this->error('Stack trace: ' . $e->getTraceAsString());
85+
}
86+
}
87+
}

‎src/Exporter/Exporter.php

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ abstract class Exporter implements FromCollection, ShouldAutoSize, WithColumnFor
3535

3636
protected string $url;
3737

38+
protected bool $optimizeMemory = true;
39+
40+
protected int $memoryCheckInterval = 1000;
41+
3842
/**
3943
* @return \Botble\DataSynchronize\Exporter\ExportColumn[]
4044
*/
@@ -204,6 +208,10 @@ public function export(): BinaryFileResponse
204208
{
205209
BaseHelper::maximumExecutionTimeAndMemoryLimit();
206210

211+
if ($this->optimizeMemory) {
212+
$this->configureMemoryOptimization();
213+
}
214+
207215
$writeType = match ($this->format) {
208216
'csv' => Excel::CSV,
209217
'xlsx' => Excel::XLSX,
@@ -271,4 +279,33 @@ public function allColumnsIsDisabled(): bool
271279
{
272280
return count($this->getAcceptedColumns()) === count(array_filter($this->getAcceptedColumns(), fn (ExportColumn $column) => $column->isDisabled()));
273281
}
282+
283+
protected function configureMemoryOptimization(): void
284+
{
285+
if ($memoryLimit = config('packages.data-synchronize.export.memory_limit')) {
286+
ini_set('memory_limit', $memoryLimit);
287+
}
288+
289+
if ($timeLimit = config('packages.data-synchronize.export.time_limit')) {
290+
set_time_limit($timeLimit);
291+
}
292+
293+
if (class_exists('DB')) {
294+
\DB::disableQueryLog();
295+
}
296+
}
297+
298+
public function setOptimizeMemory(bool $optimize): self
299+
{
300+
$this->optimizeMemory = $optimize;
301+
302+
return $this;
303+
}
304+
305+
public function setMemoryCheckInterval(int $interval): self
306+
{
307+
$this->memoryCheckInterval = $interval;
308+
309+
return $this;
310+
}
274311
}

0 commit comments

Comments
 (0)