Upgrade to Django 1.9

UPGRADING.md

@@ -19,6 +19,47 @@ Done!
 The following instructions describe modifications to the standard upgrade process required due to
 specific changes. Items are listed in reverse chronological order.
 
+## 4/28/16 - Upgrade Django from 1.7 to 1.9
+
+The time has come to upgrade Django and other required apps to their newest versions. If you are
+installing Roundware from scratch, there is no need to take extra steps. However, if your
+installation is based on a commit prior to the Django 1.9 migration, some manual setup is needed.
+
+As part of the upgrade process, [django-guardian](http://django-guardian.readthedocs.io/en/stable/)
+must be updated from 1.2.4 to 1.4.4. While the old versions of guardian used `syncdb` to populate
+the database, newer versions use migrations. The database structures are identical; however, these
+migrations do not check if the tables and relationships are already in place. Therefore, you must
+suppress guardian's initial migration.
+
+1. Make sure that you are in `su - roundware` or `su - vagrant` as appropriate
+2. Pull the relevant post-upgrade roundware-server commit (or newer)
+3. Run `pip install -r ~/roundware-server/requirements/dev.txt`
+   This will download the new requirements, both `common` and `dev`
+   Alternatively, you can `./scripts/runserver.sh` and then `Ctrl+C`
+   It should warn you about unapplied migrations
+4. Navigate to the `roundware` folder in your terminal
+5. Run `python manage.py migrate guardian --fake-initial`
+6. Run `python manage.py migrate` for the other migrations
+
+Note that you might have to also manually uninstall `django-chartit`. `django-chartit2` is meant
+to be a drop-in replacement, but if `django-chartit` is still installed, it will not work. Ensure
+that the `roundware` user can access all new `site-packages`.
+
+Next, you must run the following commands:
+
+ ```
+ rm -f /etc/apache2/sites-available/roundware.conf
+ sed s/USERNAME/roundware/g /var/www/roundware/source/files/etc-apache2-sites-available-roundware > /etc/apache2/sites-available/roundware.conf
+ # If you are using vagrant, replace roundware with vagrant after USERNAME
+ ```
+
+Roundware's `wsgi.py` was moved in this commit, and the Apache conf file must be updated. If you
+get 404 errors after upgrading, chances are you skipped this step.
+
+Lastly, ensure that you are not in a virtual environment, and run `./deploy.sh`
+Check the log to ensure that all of the required apps are located in `/var/www/roundware/lib/`
+It isn't necessary, but it might save you some headache with permissions.
+
 ### 3/7/16 - Convert from MySQL to Postgresql for GIS speaker upgrades
 Related Github issue: https://github.com/roundware/roundware-server/pull/270
 

files/etc-apache2-sites-available-roundware

@@ -36,7 +36,7 @@
   WSGIDaemonProcess roundware user=USERNAME group=USERNAME umask=002
   WSGIApplicationGroup %{GLOBAL}
   WSGIProcessGroup roundware
-  WSGIScriptAlias / /var/www/roundware/source/files/roundware.wsgi
+  WSGIScriptAlias / /var/www/roundware/source/roundware/wsgi.py
   WSGIPassAuthorization On
 
   # allow CORS for listen map, session map etc

requirements/common.txt

@@ -1,14 +1,14 @@
-Django<1.8
+Django==1.9
 # Creates REST APIs
-djangorestframework==3.2.2
+djangorestframework==3.3.3
 # Used for DRF filtering
-django-filter==0.9
+django-filter==0.13
 # Used in roundware/rw/chart_functions.py
-django-chartit==0.1
+django_chartit2
 # Used in roundware/rw/admin.py, roundware/rw/forms.py, roundware/rw/views, and more.
-django-guardian==1.2.4
+django-guardian==1.4.4
 # Used by roundware/api1/commands.py
-psutil==2.1.3
+psutil==3.4.2
 # Used by roundwared/db.py
 django-cache-utils==0.7.2
 # Used by roundware/rw/fields.py
@@ -20,9 +20,9 @@ django-braces==1.4.0
 # Loaded in roundware/urls.py
 django-adminplus==0.2.1
 # Used in roundware/rw/forms.py
-django-crispy-forms==1.4.0
+django-crispy-forms==1.6.0
 # Used in roundware/rw/widgets.py
-django-floppyforms==1.2
+django-floppyforms==1.6.1
 # Used in roundware/rw/views.py:
 django-extra-views==0.6.5
 # Used in roundware/rw/fields.py and roundware/rw/widgets.py
@@ -42,6 +42,6 @@ django-cors-headers
 # fiona is a useful tool for processing geographic files (ETL)
 fiona
 # geographic extensions for djangorestframework-gis
-djangorestframework-gis
+djangorestframework-gis==0.10.1
 # leaflet map utilities for django admin
 django-leaflet

requirements/dev.txt

@@ -1,15 +1,15 @@
 -r ./common.txt
 
-django-debug-toolbar
+django-debug-toolbar==1.4
 django-profiler==2.0
 
-#testing.  
-coverage==3.7.1
+#testing.
+coverage==4.0.3
 webtest==2.0.9
-django-webtest==1.7.5
-model_mommy==1.2
-mock==1.0.1
-python-dbusmock==0.8
+django-webtest==1.7.9
+model_mommy==1.2.6
+mock==2.0.0
+python-dbusmock==0.16.3
 
 #interactive interpreter for debugging
-ipython
+ipython==4.0.3

roundware/api2/__init__.py

@@ -1,20 +1 @@
-# Roundware Server is released under the GNU Affero General Public License v3.
-# See COPYRIGHT.txt, AUTHORS.txt, and LICENSE.txt in the project root directory.
-
-# Contains Roundware DRF REST API V2 signals.
-from __future__ import unicode_literals
-from django.contrib.auth import get_user_model
-from django.db.models.signals import post_save
-from rest_framework.authtoken.models import Token
-import logging
-
-logger = logging.getLogger(__name__)
-
-def create_auth_token(sender, instance=None, created=False, **kwargs):
-    """
-    Create an access Token for every new user
-    """
-    if created:
-        Token.objects.create(user=instance)
-
-post_save.connect(create_auth_token, get_user_model)
+default_app_config = 'api2.apps.RoundwareApi2Config'

roundware/api2/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+class RoundwareApi2Config(AppConfig):
+    name = 'api2'
+
+    def ready(self):
+        import roundware.api2.signals

roundware/api2/signals.py

@@ -0,0 +1,20 @@
+# Roundware Server is released under the GNU Affero General Public License v3.
+# See COPYRIGHT.txt, AUTHORS.txt, and LICENSE.txt in the project root directory.
+
+# Contains Roundware DRF REST API V2 signals.
+from __future__ import unicode_literals
+from django.contrib.auth import get_user_model
+from django.db.models.signals import post_save
+from rest_framework.authtoken.models import Token
+import logging
+
+logger = logging.getLogger(__name__)
+
+def create_auth_token(sender, instance=None, created=False, **kwargs):
+    """
+    Create an access Token for every new user
+    """
+    if created:
+        Token.objects.create(user=instance)
+
+post_save.connect(create_auth_token, get_user_model)

roundware/rw/filters.py

@@ -3,7 +3,7 @@
 
 from __future__ import unicode_literals
 from django.contrib.admin import DateFieldListFilter, RelatedFieldListFilter
-from django.contrib.admin.util import (get_model_from_relation,
+from django.contrib.admin.utils import (get_model_from_relation,
                                        prepare_lookup_value)
 from django.utils.translation import ugettext_lazy as _
 from django.utils.encoding import smart_unicode

roundware/rw/fixtures/default_auth.json

@@ -818,24 +818,6 @@
   "model": "auth.permission",
   "pk": 91
 },
-{
-  "fields": {
-    "username": "AnonymousUser",
-    "first_name": "",
-    "last_name": "",
-    "is_active": true,
-    "is_superuser": false,
-    "is_staff": false,
-    "last_login": "2015-10-08T13:02:44",
-    "groups": [],
-    "user_permissions": [],
-    "password": "",
-    "email": "",
-    "date_joined": "2015-10-08T13:02:44"
-  },
-  "model": "auth.user",
-  "pk": -1
-},
 {
   "fields": {
     "username": "round",

roundware/rw/fixtures/sample_project.json

@@ -956,15 +956,6 @@
   "model": "rw.listeninghistoryitem",
   "pk": 1
 },
-{
-  "fields": {
-    "client_type": null,
-    "user": -1,
-    "device_id": null
-  },
-  "model": "rw.userprofile",
-  "pk": 1
-},
 {
   "fields": {
     "client_type": null,

roundware/rw/migrations/0015_remove_null_from_mtm.py

@@ -0,0 +1,65 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9 on 2016-04-20 15:52
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rw', '0014__data__speaker_shape_creation'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='asset',
+            name='loc_description',
+            field=models.ManyToManyField(blank=True, to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='asset',
+            name='tags',
+            field=models.ManyToManyField(blank=True, to='rw.Tag'),
+        ),
+        migrations.AlterField(
+            model_name='masterui',
+            name='header_text_loc',
+            field=models.ManyToManyField(blank=True, to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='demo_stream_message_loc',
+            field=models.ManyToManyField(blank=True, related_name='demo_stream_msg_string', to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='legal_agreement_loc',
+            field=models.ManyToManyField(blank=True, related_name='legal_agreement_string', to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='out_of_range_message_loc',
+            field=models.ManyToManyField(blank=True, related_name='out_of_range_msg_string', to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='sharing_message_loc',
+            field=models.ManyToManyField(blank=True, related_name='sharing_msg_string', to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='loc_description',
+            field=models.ManyToManyField(blank=True, related_name='tag_desc', to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='loc_msg',
+            field=models.ManyToManyField(blank=True, to='rw.LocalizedString'),
+        ),
+        migrations.AlterField(
+            model_name='tag',
+            name='relationships',
+            field=models.ManyToManyField(blank=True, related_name='_tag_relationships_+', to='rw.Tag'),
+        ),
+    ]

roundware/rw/models.py

@@ -16,7 +16,6 @@
 from django.conf import settings
 from datetime import datetime
 from cache_utils.decorators import cached
-from django.contrib.auth.models import User
 from django.db.models.signals import post_save
 import logging
 from geopy.distance import vincenty
@@ -68,9 +67,9 @@ class Project(models.Model):
     speak_questions_dynamic = models.BooleanField(default=False)
     sharing_url = models.CharField(max_length=512)
     sharing_message_loc = models.ManyToManyField(
-        LocalizedString, related_name='sharing_msg_string', null=True, blank=True)
+        LocalizedString, related_name='sharing_msg_string', blank=True)
     out_of_range_message_loc = models.ManyToManyField(
-        LocalizedString, related_name='out_of_range_msg_string', null=True, blank=True)
+        LocalizedString, related_name='out_of_range_msg_string', blank=True)
     out_of_range_url = models.CharField(max_length=512)
     recording_radius = models.IntegerField(null=True)
     listen_enabled = models.BooleanField(default=False)
@@ -80,7 +79,7 @@ class Project(models.Model):
     reset_tag_defaults_on_startup = models.BooleanField(default=False)
     timed_asset_priority = models.BooleanField(default=True)
     legal_agreement_loc = models.ManyToManyField(
-        LocalizedString, related_name='legal_agreement_string', null=True, blank=True)
+        LocalizedString, related_name='legal_agreement_string', blank=True)
     repeat_mode = models.CharField(default=STOP, max_length=10, blank=False,
                                choices=REPEAT_MODES)
 
@@ -97,7 +96,7 @@ class Project(models.Model):
     demo_stream_enabled = models.BooleanField(default=False)
     demo_stream_url = models.CharField(max_length=512, blank=True)
     demo_stream_message_loc = models.ManyToManyField(
-        LocalizedString, related_name='demo_stream_msg_string', null=True, blank=True)
+        LocalizedString, related_name='demo_stream_msg_string', blank=True)
 
     out_of_range_distance = models.FloatField(default=1000)
 
@@ -156,11 +155,11 @@ class Tag(models.Model):
     value = models.TextField()
     description = models.TextField(null=True, blank=True)
     loc_description = models.ManyToManyField(
-        LocalizedString, null=True, blank=True, related_name='tag_desc')
-    loc_msg = models.ManyToManyField(LocalizedString, null=True, blank=True)
+        LocalizedString, blank=True, related_name='tag_desc')
+    loc_msg = models.ManyToManyField(LocalizedString, blank=True)
     data = models.TextField(null=True, blank=True)
     relationships = models.ManyToManyField(
-        'self', symmetrical=True, related_name='related_to', null=True, blank=True)
+        'self', symmetrical=True, related_name='related_to', blank=True)
     filter = models.CharField(
         max_length=255, default="", null=False, blank=True, choices=FILTERS)
 
@@ -199,7 +198,7 @@ class MasterUI(models.Model):
     )
     name = models.CharField(max_length=50)
     header_text_loc = models.ManyToManyField(
-        LocalizedString, null=True, blank=True)
+        LocalizedString, blank=True)
     ui_mode = models.CharField(default=LISTEN, max_length=6, blank=False,
                                choices=UI_MODES)
     tag_category = models.ForeignKey(TagCategory)
@@ -340,15 +339,15 @@ class Meta:
 
     created = models.DateTimeField(default=datetime.now)
     audiolength = models.BigIntegerField(null=True, blank=True)
-    tags = models.ManyToManyField(Tag, null=True, blank=True)
+    tags = models.ManyToManyField(Tag, blank=True)
     language = models.ForeignKey(Language, null=True)
     weight = models.IntegerField(
         choices=[(i, i) for i in range(0, 100)], default=50)
     mediatype = models.CharField(
         max_length=16, choices=ASSET_MEDIA_TYPES, default='audio')
     description = models.TextField(max_length=2048, blank=True)
     loc_description = models.ManyToManyField(
-        LocalizedString, null=True, blank=True)
+        LocalizedString, blank=True)
 
     # enables inline adding/editing of Assets in Envelope Admin.
     # creates a relationship of an Asset to the Envelope, in which it was
@@ -632,7 +631,7 @@ def __unicode__(self):
         return "%s: Asset id: %s: Start: %s: End: %s" % (self.id, self.asset.id, self.start, self.end)
 
 class UserProfile(models.Model):
-    user = models.OneToOneField(User)
+    user = models.OneToOneField(settings.AUTH_USER_MODEL)
     device_id = models.CharField(max_length=255, null=True)
     client_type = models.CharField(max_length=255, null=True)
 
@@ -641,7 +640,7 @@ def create_user_profile(sender, instance, created, **kwargs):
     if created:
         UserProfile.objects.get_or_create(user=instance)
 
-post_save.connect(create_user_profile, sender=User)
+post_save.connect(create_user_profile, sender=settings.AUTH_USER_MODEL)
 
 
 def get_field_names_from_model(model):