chore: Vendor JSONField to fix runtime warnings and reduce future upgrade pain (#13397)

The packaged version of this library contains metaclass warnings in
Django1.8. We could upgrade but newer versions of the library are
unwilling to serialize datetimes and decimal objects. Vendoring the
library will allow us to maintain compatibility for our use cases and
upgrade django at the same time.

We can't yet remove the jsonfield dependency as getsentry is using it.
Capture the intended behavior from the point we forked off.

Refs SEN-686

Author: markstory

src/sentry/db/models/fields/__init__.py

@@ -13,6 +13,7 @@
 from .citext import *  # NOQA
 from .encrypted import *  # NOQA
 from .foreignkey import *  # NOQA
+from .jsonfield import *  # NOQA
 from .gzippeddict import *  # NOQA
 from .node import *  # NOQA
 from .pickle import *  # NOQA

src/sentry/db/models/fields/encrypted.py

@@ -8,8 +8,8 @@
 
 from django.conf import settings
 from django.db.models import CharField, TextField
-from jsonfield import JSONField
 from picklefield.fields import PickledObjectField
+from sentry.db.models.fields.jsonfield import JSONField
 from sentry.db.models.utils import Creator
 from sentry.utils.encryption import decrypt, encrypt
 

src/sentry/db/models/fields/jsonfield.py

@@ -0,0 +1,142 @@
+from __future__ import absolute_import, unicode_literals
+import json
+import datetime
+import six
+
+from decimal import Decimal
+
+from django.core.exceptions import ValidationError
+from django.conf import settings
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from sentry.db.models.utils import Creator
+
+
+def default(o):
+    if hasattr(o, 'to_json'):
+        return o.to_json()
+    if isinstance(o, Decimal):
+        return six.text_type(o)
+    if isinstance(o, datetime.datetime):
+        if o.tzinfo:
+            return o.strftime('%Y-%m-%dT%H:%M:%S%z')
+        return o.strftime("%Y-%m-%dT%H:%M:%S")
+    if isinstance(o, datetime.date):
+        return o.strftime("%Y-%m-%d")
+    if isinstance(o, datetime.time):
+        if o.tzinfo:
+            return o.strftime('%H:%M:%S%z')
+        return o.strftime("%H:%M:%S")
+
+    raise TypeError(repr(o) + " is not JSON serializable")
+
+
+class JSONField(models.TextField):
+    """
+    A field that will ensure the data entered into it is valid JSON.
+
+    Originally from https://github.com/adamchainz/django-jsonfield/blob/0.9.13/jsonfield/fields.py
+    Adapted to fit our requirements of:
+
+    - always using a text field
+    - being able to serialize dates/decimals
+    - not emitting deprecation warnings
+    """
+    default_error_messages = {
+        'invalid': _("'%s' is not a valid JSON string.")
+    }
+    description = "JSON object"
+
+    def __init__(self, *args, **kwargs):
+        if not kwargs.get('null', False):
+            kwargs['default'] = kwargs.get('default', dict)
+        self.encoder_kwargs = {
+            'indent': kwargs.pop('indent', getattr(settings, 'JSONFIELD_INDENT', None))
+        }
+        super(JSONField, self).__init__(*args, **kwargs)
+        self.validate(self.get_default(), None)
+
+    def contribute_to_class(self, cls, name):
+        """
+        Add a descriptor for backwards compatibility
+        with previous Django behavior.
+        """
+        super(JSONField, self).contribute_to_class(cls, name)
+        setattr(cls, name, Creator(self))
+
+    def validate(self, value, model_instance):
+        if not self.null and value is None:
+            raise ValidationError(self.error_messages['null'])
+        try:
+            self.get_prep_value(value)
+        except BaseException:
+            raise ValidationError(self.error_messages['invalid'] % value)
+
+    def get_default(self):
+        if self.has_default():
+            default = self.default
+            if callable(default):
+                default = default()
+            if isinstance(default, six.string_types):
+                return json.loads(default)
+            return json.loads(json.dumps(default))
+        return super(JSONField, self).get_default()
+
+    def get_internal_type(self):
+        return 'TextField'
+
+    def db_type(self, connection):
+        return 'text'
+
+    def to_python(self, value):
+        if isinstance(value, six.string_types):
+            if value == "":
+                if self.null:
+                    return None
+                if self.blank:
+                    return ""
+            try:
+                value = json.loads(value)
+            except ValueError:
+                msg = self.error_messages['invalid'] % value
+                raise ValidationError(msg)
+        # TODO: Look for date/time/datetime objects within the structure?
+        return value
+
+    def get_db_prep_value(self, value, connection=None, prepared=None):
+        return self.get_prep_value(value)
+
+    def get_prep_value(self, value):
+        if value is None:
+            if not self.null and self.blank:
+                return ""
+            return None
+        return json.dumps(value, default=default, **self.encoder_kwargs)
+
+    def get_prep_lookup(self, lookup_type, value):
+        if lookup_type in ["exact", "iexact"]:
+            return self.to_python(self.get_prep_value(value))
+        if lookup_type == "in":
+            return [self.to_python(self.get_prep_value(v)) for v in value]
+        if lookup_type == "isnull":
+            return value
+        if lookup_type in ["contains", "icontains"]:
+            if isinstance(value, (list, tuple)):
+                raise TypeError("Lookup type %r not supported with argument of %s" % (
+                    lookup_type, type(value).__name__
+                ))
+                # Need a way co combine the values with '%', but don't escape that.
+                return self.get_prep_value(value)[1:-1].replace(', ', r'%')
+            if isinstance(value, dict):
+                return self.get_prep_value(value)[1:-1]
+            return self.to_python(self.get_prep_value(value))
+        raise TypeError('Lookup type %r not supported' % lookup_type)
+
+    def value_to_string(self, obj):
+        return self._get_val_from_obj(obj)
+
+
+if 'south' in settings.INSTALLED_APPS:
+    from south.modelsinspector import add_introspection_rules
+    add_introspection_rules([], ['^sentry\.db\.models\.fields\.JSONField'])

src/sentry/models/discoversavedquery.py

@@ -1,6 +1,6 @@
 from __future__ import absolute_import
 from django.db import models, transaction
-from jsonfield import JSONField
+from sentry.db.models.fields import JSONField
 from sentry.db.models import (
     Model, FlexibleForeignKey, sane_repr
 )

src/sentry/models/externalissue.py

@@ -2,9 +2,13 @@
 
 from django.db import models
 from django.utils import timezone
-from jsonfield import JSONField
 
-from sentry.db.models import BoundedPositiveIntegerField, Model, sane_repr
+from sentry.db.models import (
+    BoundedPositiveIntegerField,
+    JSONField,
+    Model,
+    sane_repr
+)
 
 
 class ExternalIssue(Model):

src/sentry/models/featureadoption.py

@@ -4,11 +4,16 @@
 
 from django.db import models, IntegrityError, transaction
 from django.utils import timezone
-from jsonfield import JSONField
 
 from sentry.adoption import manager
 from sentry.adoption.manager import UnknownFeature
-from sentry.db.models import (BaseManager, FlexibleForeignKey, Model, sane_repr)
+from sentry.db.models import (
+    BaseManager,
+    FlexibleForeignKey,
+    JSONField,
+    Model,
+    sane_repr
+)
 from sentry.utils import redis
 
 logger = logging.getLogger(__name__)

src/sentry/models/file.py

@@ -25,10 +25,14 @@
 from django.core.files.storage import get_storage_class
 from django.db import models, transaction, IntegrityError
 from django.utils import timezone
-from jsonfield import JSONField
 
 from sentry.app import locks
-from sentry.db.models import (BoundedPositiveIntegerField, FlexibleForeignKey, Model)
+from sentry.db.models import (
+    BoundedPositiveIntegerField,
+    FlexibleForeignKey,
+    JSONField,
+    Model
+)
 from sentry.tasks.files import delete_file as delete_file_task
 from sentry.utils import metrics
 from sentry.utils.retries import TimedRetryPolicy

src/sentry/models/grouplink.py

@@ -10,9 +10,14 @@
 from django.db import models
 from django.utils import timezone
 from django.utils.translation import ugettext_lazy as _
-from jsonfield import JSONField
 
-from sentry.db.models import Model, sane_repr, BoundedBigIntegerField, BoundedPositiveIntegerField
+from sentry.db.models import (
+    Model,
+    sane_repr,
+    BoundedBigIntegerField,
+    BoundedPositiveIntegerField,
+    JSONField,
+)
 
 
 class GroupLink(Model):

src/sentry/models/groupsnooze.py

@@ -4,10 +4,14 @@
 
 from django.db import models
 from django.utils import timezone
-from jsonfield import JSONField
 
 from sentry.db.models import (
-    BaseManager, BoundedPositiveIntegerField, FlexibleForeignKey, Model, sane_repr
+    BaseManager,
+    BoundedPositiveIntegerField,
+    FlexibleForeignKey,
+    JSONField,
+    Model,
+    sane_repr
 )
 
 

src/sentry/models/organizationonboardingtask.py

@@ -11,10 +11,10 @@
 from django.core.cache import cache
 from django.db import models, IntegrityError, transaction
 from django.utils import timezone
-from jsonfield import JSONField
 
 from sentry.db.models import (
-    BaseManager, BoundedBigIntegerField, BoundedPositiveIntegerField, FlexibleForeignKey, Model,
+    BaseManager, BoundedBigIntegerField, BoundedPositiveIntegerField,
+    FlexibleForeignKey, JSONField, Model,
     sane_repr
 )
 

src/sentry/models/projectkey.py

@@ -14,7 +14,6 @@
 from bitfield import BitField
 from uuid import uuid4
 
-from jsonfield import JSONField
 from django.conf import settings
 from django.core.urlresolvers import reverse
 from django.db import models
@@ -24,7 +23,12 @@
 
 from sentry import options
 from sentry.db.models import (
-    Model, BaseManager, BoundedPositiveIntegerField, FlexibleForeignKey, sane_repr
+    Model,
+    BaseManager,
+    BoundedPositiveIntegerField,
+    FlexibleForeignKey,
+    JSONField,
+    sane_repr
 )
 
 _uuid4_re = re.compile(r'^[a-f0-9]{32}$')

src/sentry/models/projectownership.py

@@ -2,14 +2,13 @@
 
 import operator
 
-from jsonfield import JSONField
 
 from django.db import models
 from django.db.models import Q
 from django.utils import timezone
 
 from sentry.db.models import Model, sane_repr
-from sentry.db.models.fields import FlexibleForeignKey
+from sentry.db.models.fields import FlexibleForeignKey, JSONField
 from sentry.ownership.grammar import load_schema
 from functools import reduce
 

src/sentry/models/prompts_activity.py

@@ -3,9 +3,14 @@
 from django.db import models
 from django.conf import settings
 from django.utils import timezone
-from jsonfield import JSONField
 
-from sentry.db.models import (BoundedPositiveIntegerField, FlexibleForeignKey, Model, sane_repr)
+from sentry.db.models import (
+    BoundedPositiveIntegerField,
+    FlexibleForeignKey,
+    JSONField,
+    Model,
+    sane_repr
+)
 
 
 class PromptsActivity(Model):

src/sentry/models/release.py

@@ -15,12 +15,12 @@
 from django.db import models, IntegrityError, transaction
 from django.db.models import F
 from django.utils import timezone
-from jsonfield import JSONField
 from time import time
 
 from sentry.app import locks
 from sentry.db.models import (
-    ArrayField, BoundedPositiveIntegerField, FlexibleForeignKey, Model, sane_repr
+    ArrayField, BoundedPositiveIntegerField, FlexibleForeignKey,
+    JSONField, Model, sane_repr
 )
 
 from sentry.models import CommitFileChange

src/sentry/models/repository.py

@@ -3,10 +3,14 @@
 from django.db import models
 from django.db.models.signals import pre_delete
 from django.utils import timezone
-from jsonfield import JSONField
 
 from sentry.constants import ObjectStatus
-from sentry.db.models import (BoundedPositiveIntegerField, Model, sane_repr)
+from sentry.db.models import (
+    BoundedPositiveIntegerField,
+    JSONField,
+    Model,
+    sane_repr
+)
 from sentry.db.mixin import PendingDeletionMixin, delete_pending_deletion_option
 from sentry.signals import pending_delete
 

src/sentry/models/scheduledeletion.py

@@ -4,10 +4,13 @@
 from django.db import models
 from django.db.models import get_model
 from django.utils import timezone
-from jsonfield import JSONField
 from uuid import uuid4
 
-from sentry.db.models import BoundedBigIntegerField, Model
+from sentry.db.models import (
+    BoundedBigIntegerField,
+    JSONField,
+    Model
+)
 
 
 def default_guid():

src/sentry/models/scheduledjob.py

@@ -2,9 +2,8 @@
 
 from django.db import models
 from django.utils import timezone
-from jsonfield import JSONField
 
-from sentry.db.models import (Model, sane_repr)
+from sentry.db.models import (JSONField, Model, sane_repr)
 
 
 def schedule_jobs(jobs):

src/sentry/models/widget.py

@@ -2,10 +2,15 @@
 
 from django.db import models
 from django.utils import timezone
-from jsonfield import JSONField
 
 from sentry.constants import ObjectStatus
-from sentry.db.models import BoundedPositiveIntegerField, FlexibleForeignKey, Model, sane_repr
+from sentry.db.models import (
+    BoundedPositiveIntegerField,
+    FlexibleForeignKey,
+    JSONField,
+    Model,
+    sane_repr
+)
 
 
 class TypesClass(object):