Skip to content

Implemented database cleaning #293

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lGuillaume124 merged 1 commit into from
Jul 14, 2017
Merged

Implemented database cleaning #293

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 75 additions & 16 deletions app/Controller/SongsController.php
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,30 +70,34 @@ public function import() {

// The difference between $found and $already_imported
$to_import = array_merge(array_diff($found, $already_imported));
$to_remove = array_merge(array_diff($already_imported, $found));
$to_import_count = count($to_import);
$to_remove_count = count($to_remove);
$found_count = count($found);
$diff_count = $found_count - $to_import_count;
$this->Session->write('to_import', $to_import);
$this->set(compact('to_import_count', 'diff_count'));
$this->Session->write('to_remove', $to_remove);
$this->set(compact('to_import_count', 'to_remove_count', 'diff_count'));
} elseif ($this->request->is('post')) {
$this->viewClass = 'Json';
$import_result = array();
$update_result = array();

if (Cache::read('import')) { // Read lock to avoid multiple import processes in the same time
$import_result[0]['status'] = 'ERR';
$import_result[0]['message'] = __('The import process is already running via another client or the CLI.');
$this->set(compact('import_result'));
$this->set('_serialize', array('import_result'));
$update_result[0]['status'] = 'ERR';
$update_result[0]['message'] = __('The import process is already running via another client or the CLI.');
$this->set(compact('update_result'));
$this->set('_serialize', array('update_result'));
} else {
// Write lock
Cache::write('import', true);

$to_import = $this->Session->read('to_import');
$to_remove = $this->Session->read('to_remove');
$imported = array();
$removed = array();

$i = 0;
foreach ($to_import as $file) {

if ($i >= 100) {
break;
}
Expand All @@ -103,19 +107,71 @@ public function import() {

$this->Song->create();
if (!$this->Song->save($parse_result['data'])) {
$import_result[$file]['status'] = 'ERR';
$import_result[$file]['message'] = __('Unable to save the song metadata to the database');
$update_result[$file]['status'] = 'ERR';
$update_result[$file]['message'] = __('Unable to save the song metadata to the database');
} else {
unset($parse_result['data']);
$import_result[$i]['file'] = $file;
$import_result[$i]['status'] = $parse_result['status'];
$import_result[$i]['message'] = $parse_result['message'];
$update_result[$i]['file'] = $file;
$update_result[$i]['status'] = $parse_result['status'];
$update_result[$i]['message'] = $parse_result['message'];
}

$imported [] = $file;
$i++;
}

$this->loadModel('PlaylistMembership');
$this->Song->virtualFields['files_with_cover'] = 'count(Song2.id)';

foreach ($to_remove as $file) {
if ($i >= 100) {
break;
}

$result = $this->Song->find('first', array(
'joins' => array(
array(
'table' => 'songs',
'alias' => 'Song2',
'type' => 'LEFT',
'conditions' => array('Song2.cover = Song.cover')
)
),
'fields' => array('Song.id', 'Song.cover', 'files_with_cover'),
'conditions' => array('Song.source_path' => $file)
));

// Remove song from database
$this->PlaylistMembership->deleteAll(array('PlaylistMembership.song_id' => $result["Song"]["id"]), false);

$update_result[$i]['file'] = $file;
if($this->Song->delete($result["Song"]["id"], false)) {
$update_result[$i]['status'] = "OK";
$update_result[$i]['message'] = "";
} else {
$update_result[$i]['status'] = "ERR";
$update_result[$i]['message'] = "Unable to delete song from the database"; //TODO: Should be handled with __( function
}

// Last file using this cover file
if ($result["Song"]['files_with_cover'] == 1) {
// Remove cover files from file system
if (file_exists(IMAGES.THUMBNAILS_DIR.DS . $result["Song"]["cover"])) {
unlink(IMAGES.THUMBNAILS_DIR.DS . $result["Song"]["cover"]);
}

// Remove resized cover files from file system
$resized_filename_base = explode(".", $result["Song"]["cover"])[0];
$resized_files = glob(RESIZED_DIR . $resized_filename_base . "_*");
foreach ($resized_files as $resized_file) {
unlink($resized_file);
}
}

$removed [] = $file;
$i++;
}

if ($i) {
$settings['Setting']['sync_token'] = time();
$this->Setting->save($settings);
Expand All @@ -125,10 +181,13 @@ public function import() {
Cache::delete('import');

$sync_token = $settings['Setting']['sync_token'];
$diff = array_diff($to_import, $imported);
$this->Session->write('to_import', $diff);
$this->set(compact('sync_token', 'import_result'));
$this->set('_serialize', array('sync_token', 'import_result'));

$import_diff = array_diff($to_import, $imported);
$remove_diff = array_diff($to_remove, $removed);
$this->Session->write('to_import', $import_diff);
$this->Session->write('to_remove', $remove_diff);
$this->set(compact('sync_token', 'update_result'));
$this->set('_serialize', array('sync_token', 'update_result'));
}

}
Expand Down
39 changes: 21 additions & 18 deletions app/View/Songs/import.ctp
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<?php echo $this->start('script'); ?>
<script type="text/javascript">
var files_count = <?php echo $to_import_count; ?>;
var files_imported = 0;
var files_count = <?php echo $to_import_count + $to_remove_count; ?>;
var files_updated = 0;
var lastResponse = "";
var noOutput = false;
function ajaxImport() {
Expand All @@ -10,7 +10,7 @@
if (xhr.readyState == 4) {

var sync_token = 0;
files_imported += 100;
files_updated += 100;

try {
var res = JSON.parse(xhr.response);
Expand All @@ -19,20 +19,20 @@
console.log('Unable to parse response: ' + error);
}

for (var i = 0, len = res['import_result'].length; i < len; i++) {
if (res['import_result'][i]['status'] == 'WARN') {
for (var i = 0, len = res['update_result'].length; i < len; i++) {
if (res['update_result'][i]['status'] == 'WARN') {

$('#accordion-warn').removeClass('hidden');
$('#warn-logs').append('[' + res['import_result'][i]['file'] + '] ' + res['import_result'][i]['message'] + '<br />');
$('#warn-logs').append('[' + res['update_result'][i]['file'] + '] ' + res['update_result'][i]['message'] + '<br />');

} else if (res['import_result'][i]['status'] == 'ERR') {
} else if (res['update_result'][i]['status'] == 'ERR') {

$('#import-panel').toggleClass('panel-primary panel-danger');
$('#import-panel-header').text("<?php echo __('Something bad happened, import aborted :('); ?>");
$('#import-panel-header').text("<?php echo __('Something bad happened, update aborted :('); ?>");
$('#import-progress-bar').toggleClass('progress-bar-stripped progress-bar-danger');
$('#import-panel-footer').remove();
$('#accordion-warn').removeClass('hidden');
$('#warn-logs').append('[' + res['import_result'][i]['file'] + '] ' + res['import_result'][i]['message'] + '<br />');
$('#warn-logs').append('[' + res['update_result'][i]['file'] + '] ' + res['update_result'][i]['message'] + '<br />');

songsManager.sync(res['sync_token']);
this.abort();
Expand All @@ -41,22 +41,22 @@
}
}

if (files_imported >= files_count) {
if (files_updated >= files_count) {

$('#import-panel').toggleClass('panel-primary panel-success');
$('#import-panel-header').text("<?php echo __('Import successfully done'); ?>");
$('#import-panel-header').text("<?php echo __('Update successfully done'); ?>");
$('#import-progress-bar').toggleClass('progress-bar-striped progress-bar-success').css('width', '100%').text('100%');
$('#import-panel-footer').remove();

songsManager.sync(res['sync_token']);

} else {

var percentage = Math.round(files_imported * 100 / files_count);
var percentage = Math.round(files_updated * 100 / files_count);
$('#import-progress-bar').css('width', percentage + '%').text(percentage + '%');
$('#import-last-label').removeClass('hidden');

var fullpath_last_import = res['import_result'][res['import_result'].length - 1]['file'] ? res['import_result'][res['import_result'].length - 1]['file'] : 'unknown';
var fullpath_last_import = res['update_result'][res['update_result'].length - 1]['file'] ? res['update_result'][res['update_result'].length - 1]['file'] : 'unknown';
var splitted_lat_import = fullpath_last_import.split('/');
$('#import-last').text(splitted_lat_import[splitted_lat_import.length - 1]);
ajaxImport();
Expand All @@ -69,7 +69,7 @@

$('#start-import-btn').on('click', function(e) {
e.preventDefault();
$('#import-panel-header').html('<strong>' + "<?php echo __('Import currently running. Please do not leave the page.'); ?>" + '</strong>');
$('#import-panel-header').html('<strong>' + "<?php echo __('Update currently running. Please do not leave the page.'); ?>" + '</strong>');
$('#start-import-btn').addClass('disabled').text("<?php echo __('Running...'); ?>");
ajaxImport();
});
Expand All @@ -80,7 +80,7 @@
<div class="col-lg-12">
<h3><?php echo __('Update the music collection'); ?></h3>
<hr />
<?php if ($to_import_count > 0 && !Cache::read('import')): ?>
<?php if (($to_import_count > 0 || $to_remove_count > 0) && !Cache::read('import')): ?>
<?php if ($to_import_count > 5000): ?>
<span class="help-block">
<?php echo __('Have a huge collection? You might be interested in the CLI tool'); ?>
Expand All @@ -95,6 +95,9 @@
<div class="panel panel-primary" id="import-panel">
<div class="panel-heading" id="import-panel-header">
<?php echo __n("%s song detected ", "%s songs detected ", $to_import_count, $to_import_count) . '(' . $diff_count . __(' already imported)'); ?>
<?php if ($to_remove_count > 0): ?>
<?php echo ', ' . __n("%s song removed from file system ", "%s songs removed from file system ", $to_remove_count, $to_remove_count); ?>
<?php endif; ?>
</div>
<div class="panel-body">
<div class="progress" style="margin-bottom: 0;">
Expand All @@ -105,12 +108,12 @@
<div class="panel-footer" id="import-panel-footer">
<div class="col-xs-6">
<p class="help-block hidden" id="import-last-label">
<?php echo __('Last import:') . ' '; ?><span id="import-last"></span>
<?php echo __('Last update:') . ' '; ?><span id="import-last"></span>
</p>
</div>
<div class="col-xs-6 text-right">
<button class="btn btn-info" id="start-import-btn">
<?php echo __('Start Import'); ?>
<?php echo __('Start Update'); ?>
</button>
</div>
<div class="clearfix"></div>
Expand All @@ -132,7 +135,7 @@
</div>
<?php elseif (Cache::read('import')): ?>
<div class="alert alert-warning">
<?php echo __('The import process is already running via another client or the CLI. You can click on "Clear cache" on the settings page to remove the lock, if needed.'); ?>
<?php echo __('The update process is already running via another client or the CLI. You can click on "Clear cache" on the settings page to remove the lock, if needed.'); ?>
</div>
<?php else: ?>
<div class="alert alert-info">
Expand Down