From 37803392a4fc9f8125dddc69726e4b134b8b686b Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Sun, 15 Jun 2014 14:09:46 +0200 Subject: [PATCH 01/11] Added the backupsetofrun model --- .../0006_auto__add_backupsetofrun.py | 138 ++++++++++++++++++ backups/models.py | 28 ++++ 2 files changed, 166 insertions(+) create mode 100644 backups/migrations/0006_auto__add_backupsetofrun.py diff --git a/backups/migrations/0006_auto__add_backupsetofrun.py b/backups/migrations/0006_auto__add_backupsetofrun.py new file mode 100644 index 0000000..d25b6ba --- /dev/null +++ b/backups/migrations/0006_auto__add_backupsetofrun.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'BackupSetOfRun' + db.create_table(u'backups_backupsetofrun', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('type', self.gf('django.db.models.fields.CharField')(max_length=16)), + ('status', self.gf('django.db.models.fields.CharField')(max_length=16)), + ('start_date', self.gf('django.db.models.fields.DateTimeField')()), + ('end_date', self.gf('django.db.models.fields.DateTimeField')(null=True, blank=True)), + ('total_size', self.gf('django.db.models.fields.BigIntegerField')()), + ('total_files', self.gf('django.db.models.fields.BigIntegerField')()), + )) + db.send_create_signal(u'backups', ['BackupSetOfRun']) + + # Adding M2M table for field backups on 'BackupSetOfRun' + m2m_table_name = db.shorten_name(u'backups_backupsetofrun_backups') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('backupsetofrun', models.ForeignKey(orm[u'backups.backupsetofrun'], null=False)), + ('backuprun', models.ForeignKey(orm[u'backups.backuprun'], null=False)) + )) + db.create_unique(m2m_table_name, ['backupsetofrun_id', 'backuprun_id']) + + + def backwards(self, orm): + # Deleting model 'BackupSetOfRun' + db.delete_table(u'backups_backupsetofrun') + + # Removing M2M table for field backups on 'BackupSetOfRun' + db.delete_table(db.shorten_name(u'backups_backupsetofrun_backups')) + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'backups.backup': { + 'Meta': {'object_name': 'Backup'}, + 'active': ('django.db.models.fields.BooleanField', [], {}), + 'excludes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'folder_from': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'folder_to': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'prox_and_sys_excludes': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'server_from': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_of_the_server'", 'to': u"orm['servers.Server']"}), + 'server_to': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_on_the_server'", 'to': u"orm['servers.Server']"}) + }, + u'backups.backuprun': { + 'Meta': {'object_name': 'BackupRun'}, + 'backup': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['backups.Backup']"}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'nb_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {}), + 'stderr': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'stdout': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'backups.backupsetofrun': { + 'Meta': {'object_name': 'BackupSetOfRun'}, + 'backups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['backups.BackupRun']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'max_length': '16'}), + 'total_files': ('django.db.models.fields.BigIntegerField', [], {}), + 'total_size': ('django.db.models.fields.BigIntegerField', [], {}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'servers.server': { + 'Meta': {'object_name': 'Server'}, + 'external_hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'external_interface': ('django.db.models.fields.CharField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'external_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'is_proxmox': ('django.db.models.fields.BooleanField', [], {}), + 'is_vm': ('django.db.models.fields.BooleanField', [], {}), + 'keymanger_name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'logstash_shipper': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'mysql_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'mysqled_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'ngnix_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'ngnixed_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'notes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'proxmox_node_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_base_folder': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_management': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'ssh_connection_string_from_backup': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'ssh_connection_string_from_gestion': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'users_owning_the_server': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'vm_host': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}) + } + } + + complete_apps = ['backups'] \ No newline at end of file diff --git a/backups/models.py b/backups/models.py index 898528a..b766763 100644 --- a/backups/models.py +++ b/backups/models.py @@ -42,3 +42,31 @@ def not_too_old(self): from django.utils import timezone import datetime return (self.end_date + datetime.timedelta(days=3)) > timezone.now() + + +class BackupSetOfRun(models.Model): + + TYPE_CHOICES = ( + ('hourly', 'Hourly'), + ('daily', 'Daily'), + ('weekly', 'Weekly'), + ('monthly', 'Monthly'), + ) + + type = models.CharField(max_length=16, choices=TYPE_CHOICES) + + STATUS_CHOICES = ( + ('running', 'Running'), + ('done', 'Done'), + ('canceled', 'Cancelled'), + ) + + status = models.CharField(max_length=16, choices=STATUS_CHOICES) + + start_date = models.DateTimeField() + end_date = models.DateTimeField(blank=True, null=True) + + backups = models.ManyToManyField(BackupRun) + + total_size = models.BigIntegerField() + total_files = models.BigIntegerField() From 007214583fe0b5bd2fc544cba1382c4dd92f48da Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Sun, 15 Jun 2014 14:22:35 +0200 Subject: [PATCH 02/11] Updated model with ref to backup --- backups/migrations/0007_auto.py | 124 ++++++++++++++++++++++++++++++++ backups/models.py | 9 +-- 2 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 backups/migrations/0007_auto.py diff --git a/backups/migrations/0007_auto.py b/backups/migrations/0007_auto.py new file mode 100644 index 0000000..735680e --- /dev/null +++ b/backups/migrations/0007_auto.py @@ -0,0 +1,124 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding M2M table for field backupruns on 'BackupSetOfRun' + m2m_table_name = db.shorten_name(u'backups_backupsetofrun_backupruns') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('backupsetofrun', models.ForeignKey(orm[u'backups.backupsetofrun'], null=False)), + ('backuprun', models.ForeignKey(orm[u'backups.backuprun'], null=False)) + )) + db.create_unique(m2m_table_name, ['backupsetofrun_id', 'backuprun_id']) + + + def backwards(self, orm): + # Removing M2M table for field backupruns on 'BackupSetOfRun' + db.delete_table(db.shorten_name(u'backups_backupsetofrun_backupruns')) + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'backups.backup': { + 'Meta': {'object_name': 'Backup'}, + 'active': ('django.db.models.fields.BooleanField', [], {}), + 'excludes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'folder_from': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'folder_to': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'prox_and_sys_excludes': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'server_from': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_of_the_server'", 'to': u"orm['servers.Server']"}), + 'server_to': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_on_the_server'", 'to': u"orm['servers.Server']"}) + }, + u'backups.backuprun': { + 'Meta': {'object_name': 'BackupRun'}, + 'backup': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['backups.Backup']"}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'nb_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {}), + 'stderr': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'stdout': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'backups.backupsetofrun': { + 'Meta': {'object_name': 'BackupSetOfRun'}, + 'backupruns': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['backups.BackupRun']", 'symmetrical': 'False'}), + 'backups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['backups.Backup']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'running'", 'max_length': '16'}), + 'total_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'total_size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'servers.server': { + 'Meta': {'object_name': 'Server'}, + 'external_hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'external_interface': ('django.db.models.fields.CharField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'external_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'is_proxmox': ('django.db.models.fields.BooleanField', [], {}), + 'is_vm': ('django.db.models.fields.BooleanField', [], {}), + 'keymanger_name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'logstash_shipper': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'mysql_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'mysqled_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'ngnix_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'ngnixed_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'notes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'proxmox_node_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_base_folder': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_management': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'ssh_connection_string_from_backup': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'ssh_connection_string_from_gestion': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'users_owning_the_server': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'vm_host': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}) + } + } + + complete_apps = ['backups'] \ No newline at end of file diff --git a/backups/models.py b/backups/models.py index b766763..2620403 100644 --- a/backups/models.py +++ b/backups/models.py @@ -61,12 +61,13 @@ class BackupSetOfRun(models.Model): ('canceled', 'Cancelled'), ) - status = models.CharField(max_length=16, choices=STATUS_CHOICES) + status = models.CharField(max_length=16, choices=STATUS_CHOICES, default='running') start_date = models.DateTimeField() end_date = models.DateTimeField(blank=True, null=True) - backups = models.ManyToManyField(BackupRun) + backupruns = models.ManyToManyField(BackupRun) + backups = models.ManyToManyField(Backup) - total_size = models.BigIntegerField() - total_files = models.BigIntegerField() + total_size = models.BigIntegerField(default=0) + total_files = models.BigIntegerField(default=0) From f657946e79066253654d7b4098fdb88da89f4027 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Sun, 15 Jun 2014 14:54:13 +0200 Subject: [PATCH 03/11] Use backuprun to save total time, size of a backup and prevent two hourly backup to run at the same time --- app/settings.py | 2 + app/utils.py | 20 +++ ...to__chg_field_backupsetofrun_start_date.py | 118 +++++++++++++++ backups/migrations/0009_auto.py | 134 +++++++++++++++++ backups/migrations/0010_auto.py | 136 ++++++++++++++++++ backups/models.py | 2 +- backups/tasks.py | 82 +++++++++-- 7 files changed, 485 insertions(+), 9 deletions(-) create mode 100644 app/utils.py create mode 100644 backups/migrations/0008_auto__chg_field_backupsetofrun_start_date.py create mode 100644 backups/migrations/0009_auto.py create mode 100644 backups/migrations/0010_auto.py diff --git a/app/settings.py b/app/settings.py index f966f70..1f71d8d 100644 --- a/app/settings.py +++ b/app/settings.py @@ -162,6 +162,8 @@ NGNIX_DEFAULT_REDIRECT = '' +BACKUP_SET_LOCK = '/tmp/_azimutgestion_backuplock.lock' + try: from settingsLocal import * except ImportError: diff --git a/app/utils.py b/app/utils.py new file mode 100644 index 0000000..c670764 --- /dev/null +++ b/app/utils.py @@ -0,0 +1,20 @@ +class DjangoLock: + + def __init__(self, filename): + self.filename = filename + # This will create it if it does not exist already + self.handle = open(filename, 'w') + + # flock() is a blocking call unless it is bitwise ORed with LOCK_NB to avoid blocking + # on lock acquisition. This blocking is what I use to provide atomicity across forked + # Django processes since native python locks and semaphores only work at the thread level + def acquire(self): + import fcntl + fcntl.flock(self.handle, fcntl.LOCK_EX) + + def release(self): + import fcntl + fcntl.flock(self.handle, fcntl.LOCK_UN) + + def __del__(self): + self.handle.close() diff --git a/backups/migrations/0008_auto__chg_field_backupsetofrun_start_date.py b/backups/migrations/0008_auto__chg_field_backupsetofrun_start_date.py new file mode 100644 index 0000000..4f41dcf --- /dev/null +++ b/backups/migrations/0008_auto__chg_field_backupsetofrun_start_date.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Changing field 'BackupSetOfRun.start_date' + db.alter_column(u'backups_backupsetofrun', 'start_date', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True)) + + def backwards(self, orm): + + # Changing field 'BackupSetOfRun.start_date' + db.alter_column(u'backups_backupsetofrun', 'start_date', self.gf('django.db.models.fields.DateTimeField')()) + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'backups.backup': { + 'Meta': {'object_name': 'Backup'}, + 'active': ('django.db.models.fields.BooleanField', [], {}), + 'excludes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'folder_from': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'folder_to': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'prox_and_sys_excludes': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'server_from': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_of_the_server'", 'to': u"orm['servers.Server']"}), + 'server_to': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_on_the_server'", 'to': u"orm['servers.Server']"}) + }, + u'backups.backuprun': { + 'Meta': {'object_name': 'BackupRun'}, + 'backup': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['backups.Backup']"}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'nb_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {}), + 'stderr': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'stdout': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'backups.backupsetofrun': { + 'Meta': {'object_name': 'BackupSetOfRun'}, + 'backupruns': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['backups.BackupRun']", 'symmetrical': 'False'}), + 'backups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['backups.Backup']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'running'", 'max_length': '16'}), + 'total_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'total_size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'servers.server': { + 'Meta': {'object_name': 'Server'}, + 'external_hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'external_interface': ('django.db.models.fields.CharField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'external_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'is_proxmox': ('django.db.models.fields.BooleanField', [], {}), + 'is_vm': ('django.db.models.fields.BooleanField', [], {}), + 'keymanger_name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'logstash_shipper': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'mysql_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'mysqled_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'ngnix_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'ngnixed_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'notes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'proxmox_node_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_base_folder': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_management': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'ssh_connection_string_from_backup': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'ssh_connection_string_from_gestion': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'users_owning_the_server': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'vm_host': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}) + } + } + + complete_apps = ['backups'] \ No newline at end of file diff --git a/backups/migrations/0009_auto.py b/backups/migrations/0009_auto.py new file mode 100644 index 0000000..5923abc --- /dev/null +++ b/backups/migrations/0009_auto.py @@ -0,0 +1,134 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Removing M2M table for field backups on 'BackupSetOfRun' + db.delete_table(db.shorten_name(u'backups_backupsetofrun_backups')) + + # Removing M2M table for field backupruns on 'BackupSetOfRun' + db.delete_table(db.shorten_name(u'backups_backupsetofrun_backupruns')) + + + def backwards(self, orm): + # Adding M2M table for field backups on 'BackupSetOfRun' + m2m_table_name = db.shorten_name(u'backups_backupsetofrun_backups') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('backupsetofrun', models.ForeignKey(orm[u'backups.backupsetofrun'], null=False)), + ('backup', models.ForeignKey(orm[u'backups.backup'], null=False)) + )) + db.create_unique(m2m_table_name, ['backupsetofrun_id', 'backup_id']) + + # Adding M2M table for field backupruns on 'BackupSetOfRun' + m2m_table_name = db.shorten_name(u'backups_backupsetofrun_backupruns') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('backupsetofrun', models.ForeignKey(orm[u'backups.backupsetofrun'], null=False)), + ('backuprun', models.ForeignKey(orm[u'backups.backuprun'], null=False)) + )) + db.create_unique(m2m_table_name, ['backupsetofrun_id', 'backuprun_id']) + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'backups.backup': { + 'Meta': {'object_name': 'Backup'}, + 'active': ('django.db.models.fields.BooleanField', [], {}), + 'excludes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'folder_from': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'folder_to': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'prox_and_sys_excludes': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'server_from': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_of_the_server'", 'to': u"orm['servers.Server']"}), + 'server_to': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_on_the_server'", 'to': u"orm['servers.Server']"}) + }, + u'backups.backuprun': { + 'Meta': {'object_name': 'BackupRun'}, + 'backup': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['backups.Backup']"}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'nb_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {}), + 'stderr': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'stdout': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'backups.backupsetofrun': { + 'Meta': {'object_name': 'BackupSetOfRun'}, + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'running'", 'max_length': '16'}), + 'total_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'total_size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'servers.server': { + 'Meta': {'object_name': 'Server'}, + 'external_hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'external_interface': ('django.db.models.fields.CharField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'external_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'is_proxmox': ('django.db.models.fields.BooleanField', [], {}), + 'is_vm': ('django.db.models.fields.BooleanField', [], {}), + 'keymanger_name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'logstash_shipper': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'mysql_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'mysqled_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'ngnix_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'ngnixed_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'notes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'proxmox_node_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_base_folder': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_management': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'ssh_connection_string_from_backup': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'ssh_connection_string_from_gestion': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'users_owning_the_server': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'vm_host': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}) + } + } + + complete_apps = ['backups'] \ No newline at end of file diff --git a/backups/migrations/0010_auto.py b/backups/migrations/0010_auto.py new file mode 100644 index 0000000..8f50fd7 --- /dev/null +++ b/backups/migrations/0010_auto.py @@ -0,0 +1,136 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding M2M table for field backupruns on 'BackupSetOfRun' + m2m_table_name = db.shorten_name(u'backups_backupsetofrun_backupruns') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('backupsetofrun', models.ForeignKey(orm[u'backups.backupsetofrun'], null=False)), + ('backuprun', models.ForeignKey(orm[u'backups.backuprun'], null=False)) + )) + db.create_unique(m2m_table_name, ['backupsetofrun_id', 'backuprun_id']) + + # Adding M2M table for field backups on 'BackupSetOfRun' + m2m_table_name = db.shorten_name(u'backups_backupsetofrun_backups') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('backupsetofrun', models.ForeignKey(orm[u'backups.backupsetofrun'], null=False)), + ('backup', models.ForeignKey(orm[u'backups.backup'], null=False)) + )) + db.create_unique(m2m_table_name, ['backupsetofrun_id', 'backup_id']) + + + def backwards(self, orm): + # Removing M2M table for field backupruns on 'BackupSetOfRun' + db.delete_table(db.shorten_name(u'backups_backupsetofrun_backupruns')) + + # Removing M2M table for field backups on 'BackupSetOfRun' + db.delete_table(db.shorten_name(u'backups_backupsetofrun_backups')) + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + u'backups.backup': { + 'Meta': {'object_name': 'Backup'}, + 'active': ('django.db.models.fields.BooleanField', [], {}), + 'excludes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'folder_from': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'folder_to': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'prox_and_sys_excludes': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'server_from': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_of_the_server'", 'to': u"orm['servers.Server']"}), + 'server_to': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'backups_on_the_server'", 'to': u"orm['servers.Server']"}) + }, + u'backups.backuprun': { + 'Meta': {'object_name': 'BackupRun'}, + 'backup': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['backups.Backup']"}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'nb_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {}), + 'stderr': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'stdout': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}) + }, + u'backups.backupsetofrun': { + 'Meta': {'object_name': 'BackupSetOfRun'}, + 'backupruns': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['backups.BackupRun']", 'symmetrical': 'False'}), + 'backups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['backups.Backup']", 'symmetrical': 'False'}), + 'end_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'start_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.CharField', [], {'default': "'running'", 'max_length': '16'}), + 'total_files': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'total_size': ('django.db.models.fields.BigIntegerField', [], {'default': '0'}), + 'type': ('django.db.models.fields.CharField', [], {'max_length': '16'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'servers.server': { + 'Meta': {'object_name': 'Server'}, + 'external_hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'external_interface': ('django.db.models.fields.CharField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'external_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'hostname_for_vms_creation': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'internal_ip': ('django.db.models.fields.IPAddressField', [], {'max_length': '15', 'null': 'True', 'blank': 'True'}), + 'is_proxmox': ('django.db.models.fields.BooleanField', [], {}), + 'is_vm': ('django.db.models.fields.BooleanField', [], {}), + 'keymanger_name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'logstash_shipper': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'mysql_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'mysqled_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'ngnix_server': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'ngnixed_server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}), + 'notes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'proxmox_node_name': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_base_folder': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'samba_management': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'ssh_connection_string_from_backup': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'ssh_connection_string_from_gestion': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'users_owning_the_server': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': u"orm['auth.User']", 'null': 'True', 'blank': 'True'}), + 'vm_host': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'server_set'", 'null': 'True', 'to': u"orm['servers.Server']"}) + } + } + + complete_apps = ['backups'] \ No newline at end of file diff --git a/backups/models.py b/backups/models.py index 2620403..ee2af89 100644 --- a/backups/models.py +++ b/backups/models.py @@ -63,7 +63,7 @@ class BackupSetOfRun(models.Model): status = models.CharField(max_length=16, choices=STATUS_CHOICES, default='running') - start_date = models.DateTimeField() + start_date = models.DateTimeField(auto_now_add=True) end_date = models.DateTimeField(blank=True, null=True) backupruns = models.ManyToManyField(BackupRun) diff --git a/backups/tasks.py b/backups/tasks.py index 78c6f4b..5b92adb 100644 --- a/backups/tasks.py +++ b/backups/tasks.py @@ -3,14 +3,16 @@ import re from django.utils import timezone -from backups.models import Backup, BackupRun +from backups.models import Backup, BackupRun, BackupSetOfRun import os from django.conf import settings +from app.utils import DjangoLock + @task(ignore_result=True) -def run_backup(id, mode='hourly'): +def run_backup(id, mode='hourly', backupsetpk=None): """Run a backup""" backup = Backup.objects.get(pk=id) @@ -19,13 +21,19 @@ def run_backup(id, mode='hourly'): backuprun = BackupRun(backup=backup, start_date=timezone.now()) backuprun.save() + def _notify_set_if_needed(): + if backupsetpk: + check_end_of_backupset.delay(backupsetpk, backuprun.pk) + # Backup if not backup.server_to.ssh_connection_string_from_gestion: - print "Error: No connection from gestion" + print("Error: No connection from gestion") + _notify_set_if_needed() return if not backup.server_from.ssh_connection_string_from_backup: - print "Error: No connection from backup" + print("Error: No connection from backup") + _notify_set_if_needed() return os.system('ssh ' + backup.server_to.ssh_connection_string_from_gestion + ' wget ' + settings.GESTION_URL + 'backups/get_conf/' + str(backup.pk) + '/ -O /tmp/azimut-gestion-backup-config-' + str(backup.pk)) @@ -37,6 +45,10 @@ def run_backup(id, mode='hourly'): out, err = p.communicate() + # import time + # time.sleep(10) + # out, err = "1", "2" + backuprun.end_date = timezone.now() backuprun.stdout = out backuprun.stderr = err @@ -44,20 +56,74 @@ def run_backup(id, mode='hourly'): try: backuprun.nb_files = re.search('Number of files: (\d*)', out).group(1) or 0 except: - print "Error getting nb files" + print("Error getting nb files") pass try: backuprun.size = re.search('Total file size: (\d*)', out).group(1) or 0 except: - print "Error getting total file size" + print("Error getting total file size") pass backuprun.save() + _notify_set_if_needed() + @task(ignore_result=True) def run_active_backups(mode): """Run all actives backups""" - for bkp in Backup.objects.filter(active=True).all(): - run_backup.delay(bkp.pk, mode) + # If we're in hourly mode, check that the previous backup was done. (In + # othercases, it's ok to run more than one at a time) + if mode == 'hourly': + oldbackupruns = BackupSetOfRun.objects.filter(type='hourly', status='running').all() + + if oldbackupruns: + # TODO: Create a warning + print("Aborting run as there is still backup runnning !") + return + + backups_to_run = Backup.objects.filter(active=True).all() + + backupset = BackupSetOfRun(type=mode) + backupset.save() + + for bpk in backups_to_run: + backupset.backups.add(bpk) + + backupset.save() + + for bkp in backups_to_run: + run_backup.delay(bkp.pk, mode, backupset.pk) + + +@task(ignore_result=True) +def check_end_of_backupset(backupsetpk, backuprunpk): + + l = DjangoLock(settings.BACKUP_SET_LOCK) + + l.acquire() + + backupset = BackupSetOfRun.objects.get(pk=backupsetpk) + backuprun = BackupRun.objects.get(pk=backuprunpk) + + try: + + backupset.backups.remove(backuprun.backup) + backupset.backupruns.add(backuprun) + + backupset.total_size += backuprun.size + backupset.total_files += backuprun.nb_files + + if not backupset.backups.all(): # End :) + backupset.end_date = timezone.now() + backupset.status = 'done' + + # TODO: Send mail ? + + backupset.save() + + except Exception as e: + print("Error during check end of backup set !" + str(e)) + + l.release() From e6c012dda7628f791cb61d89fa63d2fc761ba616 Mon Sep 17 00:00:00 2001 From: Maximilien Cuony Date: Sun, 15 Jun 2014 16:31:50 +0200 Subject: [PATCH 04/11] List of backup sets, cleaup, cancel --- backups/models.py | 27 ++++++++ backups/templates/backups/backups/list.html | 2 +- .../templates/backups/backupsets/list.html | 64 +++++++++++++++++++ backups/urls.py | 3 + backups/views.py | 45 ++++++++++++- templates/base.html | 5 +- 6 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 backups/templates/backups/backupsets/list.html diff --git a/backups/models.py b/backups/models.py index ee2af89..d85a6f7 100644 --- a/backups/models.py +++ b/backups/models.py @@ -1,6 +1,7 @@ from django.db import models from servers.models import Server +from django.utils import timezone class Backup(models.Model): @@ -71,3 +72,29 @@ class BackupSetOfRun(models.Model): total_size = models.BigIntegerField(default=0) total_files = models.BigIntegerField(default=0) + + def get_status_label(self): + VALUES = {'running': 'warning', 'done': 'success', 'canceled': 'important'} + + if self.status in VALUES: + return VALUES[self.status] + else: + return 'important' + + def get_total_time(self): + if not self.end_date: + end_date = timezone.now() + else: + end_date = self.end_date + + return int((end_date - self.start_date).total_seconds() / 36.0) / 100.0 + + def get_total_time_label(self): + + tt = self.get_total_time() + + if tt < 3.5: + return 'success' + if tt < 4.0: + return 'warning' + return 'important' diff --git a/backups/templates/backups/backups/list.html b/backups/templates/backups/backups/list.html index f6ab0f1..4da4aad 100644 --- a/backups/templates/backups/backups/list.html +++ b/backups/templates/backups/backups/list.html @@ -72,7 +72,7 @@

{% trans "Backups" %}

- {% trans "Cleanup >24h runs" %} + {% trans "Cleanup >24h runs not in a set" %} {% trans "Add" %}
diff --git a/backups/templates/backups/backupsets/list.html b/backups/templates/backups/backupsets/list.html new file mode 100644 index 0000000..f43b92c --- /dev/null +++ b/backups/templates/backups/backupsets/list.html @@ -0,0 +1,64 @@ +{% extends "base.html" %} +{% load i18n %} + + +{% block title %}Backup sets{% endblock %} + +{% block content %} + +

{% trans "Backups sets" %}

+ + + +
+ +
+
+
{% trans "List of backups sets" %}
+
+
+
+ + + + + + + + + + + + + + + + {% for elem in liste %} + + + + + + + + + + + {% endfor %} + +
{% trans "Start date" %}{% trans "End date" %}{% trans "Type" %}{% trans "Status" %}{% trans "Total time" %}{% trans "Total size" %}{% trans "Total files" %}
{{elem.start_date}}{{elem.end_date}} ({{elem.end_date|timesince}}){{elem.get_type_display}}{{elem.get_status_display}}{{elem.get_total_time}} hours{% if elem.type == 'hourly' %}{{elem.total_size|filesizeformat}}{% endif %}{% if elem.type == 'hourly' %}{{elem.total_files}}{% endif %} + {% if elem.status == 'running' %} + {% trans "Set as canceled" %} + {% endif %} +
+ +
+ {% trans "Cleanup olds sets" %} +
+ +
+
+
+
+ +{% endblock %} diff --git a/backups/urls.py b/backups/urls.py index 99492f4..1c8c2a3 100644 --- a/backups/urls.py +++ b/backups/urls.py @@ -13,5 +13,8 @@ url(r'^get_conf/(?P[0-9]*)/$', 'get_conf'), url(r'^clean_up$', 'clean_up'), + url(r'^clean_up_old_sets$', 'clean_up_old_sets'), + url(r'^runs$', 'backupsets_list'), + url(r'^runs/(?P[0-9]*)/cancel$', 'backupsets_cancel'), ) diff --git a/backups/views.py b/backups/views.py index f74e0b7..76cee59 100644 --- a/backups/views.py +++ b/backups/views.py @@ -16,7 +16,7 @@ from django.core.urlresolvers import reverse from django.contrib import messages -from backups.models import Backup, BackupRun +from backups.models import Backup, BackupRun, BackupSetOfRun from backups.forms import BackupForm from backups.tasks import run_backup @@ -167,8 +167,49 @@ def get_conf(request, pk): @staff_member_required def clean_up(request): - BackupRun.objects.filter(start_date__lt=timezone.now() - datetime.timedelta(days=1)).delete() + for b in BackupRun.objects.filter(start_date__lt=timezone.now() - datetime.timedelta(days=1)).all(): + if b.backupsetofrun_set.count() == 0: + b.delete() messages.success(request, "Old backups runs have been deleted") return HttpResponseRedirect(reverse('backups.views.backups_list')) + + +@login_required +@staff_member_required +def backupsets_list(request): + """Show the list of backup sets""" + + liste = BackupSetOfRun.objects.order_by('-start_date').all() + + return render_to_response('backups/backupsets/list.html', {'liste': liste}, context_instance=RequestContext(request)) + + +@login_required +@staff_member_required +def backupsets_cancel(request, pk): + + backupset = get_object_or_404(BackupSetOfRun, pk=pk, status='running') + backupset.status = 'canceled' + backupset.end_date = timezone.now() + backupset.save() + + return HttpResponseRedirect(reverse('backups.views.backupsets_list')) + + +@login_required +@staff_member_required +def clean_up_old_sets(request): + + NB_OK = {'hourly': 6, 'daily': 7, 'weekly': 7, 'monthly': 3} + + for b in BackupSetOfRun.objects.order_by('-start_date').all(): + if NB_OK[b.type] >= 0: + NB_OK[b.type] -= 1 + else: + b.delete() + + messages.success(request, "Old backups sets have been deleted") + + return HttpResponseRedirect(reverse('backups.views.backupsets_list')) diff --git a/templates/base.html b/templates/base.html index 16e6d86..4539a7b 100644 --- a/templates/base.html +++ b/templates/base.html @@ -118,6 +118,9 @@ + +