Merge branch 'master' of git://github.com/django-mongodb-engine/mongodb-engine

Author: key

AUTHORS.rst

@@ -13,6 +13,7 @@ Past Primary Authors
 
 Contributions by
 ----------------
+* Ellie Frost (https://github.com/stillinbeta)
 * Dag Stockstad (https://github.com/dstockstad) -- found tons of bugs in our query generator
 * Shane R. Spencer (https://github.com/whardier) -- Website Review
 * Sabin Iacob (https://github.com/m0n5t3r)

django_mongodb_engine/compiler.py

@@ -403,6 +403,7 @@ def execute_sql(self, result_type=MULTI):
             ret.append(result)
         return ret
 
+
 class SQLInsertCompiler(NonrelInsertCompiler, SQLCompiler):
     @safe_call
     def insert(self, data, return_id=False):
@@ -412,39 +413,21 @@ def insert(self, data, return_id=False):
             pass
         return self._save(data, return_id)
 
+
 # TODO: Define a common nonrel API for updates and add it to the nonrel
 # backend base classes and port this code to that API
 class SQLUpdateCompiler(NonrelUpdateCompiler, SQLCompiler):
     query_class = MongoQuery
 
-    @safe_call
-    def execute_raw(self, update_spec, multi=True, **kwargs):
-        collection = self.get_collection()
-        criteria = self.build_query()._mongo_query
-        options = self.connection.operation_flags.get('update', {})
-        options = dict(options, **kwargs)
-        info = collection.update(criteria, update_spec, multi=multi, **options)
-        if info is not None:
-            return info.get('n')
-
-    def execute_sql(self, result_type):
-        return self.execute_raw(*self._get_update_spec())
-
-    def _get_update_spec(self):
+    def update(self, values):
         multi = True
         spec = {}
-        for field, _, value in self.query.values:
+        for field, value in values:
             if getattr(field, 'forbids_updates', False):
                 raise DatabaseError("Updates on %ss are not allowed" %
                                     field.__class__.__name__)
             if field.unique:
                 multi = False
-            if hasattr(value, 'prepare_database_save'):
-                value = value.prepare_database_save(field)
-            else:
-                value = field.get_db_prep_save(value, connection=self.connection)
-
-            value = self.convert_value_for_db(field.db_type(connection=self.connection), value)
             if hasattr(value, "evaluate"):
                 assert value.connector in (value.ADD, value.SUB)
                 assert not value.negated
@@ -465,7 +448,18 @@ def _get_update_spec(self):
                 raise DatabaseError("Can not modify _id")
             spec.setdefault(action, {})[column] = value
 
-        return spec, multi
+        return self.execute_update(spec, multi)
+
+    @safe_call
+    def execute_update(self, update_spec, multi=True, **kwargs):
+        collection = self.get_collection()
+        criteria = self.build_query()._mongo_query
+        options = self.connection.operation_flags.get('update', {})
+        options = dict(options, **kwargs)
+        info = collection.update(criteria, update_spec, multi=multi, **options)
+        if info is not None:
+            return info.get('n')
+
 
 class SQLDeleteCompiler(NonrelDeleteCompiler, SQLCompiler):
     pass

django_mongodb_engine/contrib/__init__.py

@@ -46,7 +46,7 @@ def raw_update(self, spec_or_q, update_dict, **kwargs):
             queryset = self.filter(spec_or_q)
         queryset._for_write = True
         compiler = _compiler_for_queryset(queryset, 'SQLUpdateCompiler')
-        compiler.execute_raw(update_dict, **kwargs)
+        compiler.execute_update(update_dict, **kwargs)
 
     raw_update.alters_data = True
 
@@ -131,6 +131,10 @@ def inline_map_reduce(self, *args, **kwargs):
     def _get_query(self):
         return _compiler_for_queryset(self).build_query()
 
+    def distinct(self, *args, **kwargs):
+        query = self._get_query()
+        return query.collection.distinct(*args, **kwargs)
+
 class MongoDBManager(models.Manager, RawQueryMixin):
     """
     Lets you use Map/Reduce and raw query/update with your models::
@@ -147,3 +151,6 @@ def inline_map_reduce(self, *args, **kwargs):
 
     def get_query_set(self):
         return MongoDBQuerySet(self.model, using=self._db)
+
+    def distinct(self, *args, **kwargs):
+        return self.get_query_set().distinct(*args, **kwargs)

django_mongodb_engine/creation.py

@@ -2,12 +2,13 @@
 from djangotoolbox.db.base import NonrelDatabaseCreation
 from .utils import make_index_list
 
+
 class DatabaseCreation(NonrelDatabaseCreation):
     data_types = dict(NonrelDatabaseCreation.data_types, **{
         'AutoField': 'objectid',
         'ForeignKey': 'objectid',
         'OneToOneField': 'objectid',
-        'RelatedAutoField' : 'objectid',
+        'RelatedAutoField': 'objectid',
         'DecimalField': 'float',
     })
 
@@ -34,6 +35,8 @@ def ensure_index(*args, **kwargs):
             self._handle_oldstyle_indexes(ensure_index, meta)
 
     def _handle_newstyle_indexes(self, ensure_index, meta, indexes):
+        from djangotoolbox.fields import AbstractIterableField, EmbeddedModelField
+
         # Django indexes
         for field in meta.local_fields:
             if not (field.unique or field.db_index):
@@ -44,21 +47,35 @@ def _handle_newstyle_indexes(self, ensure_index, meta, indexes):
 
         # Django unique_together indexes
         indexes = list(indexes)
+
         for fields in getattr(meta, 'unique_together', []):
             assert isinstance(fields, (list, tuple))
             indexes.append({'fields': make_index_list(fields), 'unique': True})
 
+        def get_column_name(field):
+            opts = meta
+            parts = field.split('.')
+            for i, part in enumerate(parts):
+                field = opts.get_field(part)
+                parts[i] = field.column
+                if isinstance(field, AbstractIterableField):
+                    field = field.item_field
+                if isinstance(field, EmbeddedModelField):
+                    opts = field.embedded_model._meta
+                else:
+                    break
+            return '.'.join(parts)
+
         for index in indexes:
             if isinstance(index, dict):
                 kwargs = index.copy()
                 fields = kwargs.pop('fields')
             else:
                 fields, kwargs = index, {}
-            fields = [(meta.get_field(name).column, direction)
+            fields = [(get_column_name(name), direction)
                       for name, direction in make_index_list(fields)]
             ensure_index(fields, **kwargs)
 
-
     def _handle_oldstyle_indexes(self, ensure_index, meta):
         from warnings import warn
         warn("'descending_indexes', 'sparse_indexes' and 'index_together' are"

django_mongodb_engine/storage.py

@@ -1,9 +1,12 @@
 import os
+import urlparse
 
 from gridfs import GridFS, NoFile
 
 from django.core.exceptions import ImproperlyConfigured
 from django.core.files.storage import Storage
+from django.utils.encoding import filepath_to_uri
+
 
 def _get_subcollections(collection):
     """
@@ -15,6 +18,7 @@ def _get_subcollections(collection):
         if cleaned != collection.name and cleaned.startswith(collection.name):
             yield cleaned
 
+
 class GridFSStorage(Storage):
     """
     GridFS Storage backend for Django.
@@ -39,15 +43,24 @@ class GridFSStorage(Storage):
     :param database:
         Alias of the Django database to use. Defaults to 'default' (the default
         Django database).
+    :param base_url:
+        URL that serves the files in GridFS (for instance, through nginx-gridfs).
+        Defaults to None (file not accessible through a URL).
     """
 
-    def __init__(self, location='', collection='storage', database='default'):
+    def __init__(self, location='', collection='storage', database='default',
+                 base_url=None):
         self.location = location.strip(os.sep)
         self.collection = collection
         self.database = database
+        self.base_url = base_url
+
         if not self.collection:
             raise ImproperlyConfigured("'collection' may not be empty")
 
+        if self.base_url and not self.base_url.endswith('/'):
+            raise ImproperlyConfigured("If set, 'base_url' must end with a slash")
+
     def _open(self, path, mode='rb'):
         """
         Returns a :class:`~gridfs.GridOut` file opened in `mode`, or raises
@@ -104,6 +117,11 @@ def size(self, path):
         gridfs, filename = self._get_gridfs(path)
         return gridfs.get_last_version(filename=filename).length
 
+    def url(self, name):
+        if self.base_url is None:
+            raise ValueError("This file is not accessible via a URL.")
+        return urlparse.urljoin(self.base_url, filepath_to_uri(name))
+
     def created_time(self, path):
         """
         Returns the datetime the file at `path` was created.

django_mongodb_engine/utils.py

@@ -1,6 +1,7 @@
 import re
 import time
 from pymongo import ASCENDING
+from pymongo.cursor import Cursor
 from django.conf import settings
 from django.db.backends.util import logger
 
@@ -39,29 +40,56 @@ def __init__(self, collection, db_alias):
     def __getattr__(self, attr):
         return getattr(self.collection, attr)
 
-    def logging_wrapper(method, npositional=1):
+    def profile_call(self, func, args=(), kwargs={}):
+        start = time.time()
+        retval = func(*args, **kwargs)
+        duration = time.time() - start
+        return duration, retval
+
+    def log(self, op, duration, args, kwargs=None):
+        args = ' '.join(str(arg) for arg in args)
+        msg = '%s.%s (%.2f) %s' % (self.collection.name, op, duration, args)
+        kwargs = dict((k, v) for k, v in kwargs.iteritems() if v)
+        if kwargs:
+            msg += ' %s' % kwargs
+        if len(settings.DATABASES) > 1:
+            msg = self.alias + '.' + msg
+        logger.debug(msg, extra={'duration' : duration})
+
+    def find(self, *args, **kwargs):
+        if not 'slave_okay' in kwargs and self.collection.slave_okay:
+            kwargs['slave_okay'] = True
+        return DebugCursor(self, self.collection, *args, **kwargs)
+
+    def logging_wrapper(method):
         def wrapper(self, *args, **kwargs):
-            if npositional is not None:
-                assert len(args) == npositional
-            start = time.time()
-            try:
-                result = getattr(self.collection, method)(*args, **kwargs)
-            finally:
-                duration = time.time() - start
-                msg = '%s.%s (%.3f) %s' % (self.collection.name, method, duration,
-                                           ' '.join(str(arg) for arg in args))
-                if any(kwargs.itervalues()):
-                    msg += ' %s' % kwargs
-                if len(settings.DATABASES) > 1:
-                    msg = self.alias + '.' + msg
-                logger.debug(msg, extra={'duration' : duration})
-            return result
+            func = getattr(self.collection, method)
+            duration, retval = self.profile_call(func, args, kwargs)
+            self.log(method, duration, args, kwargs)
+            return retval
         return wrapper
 
-    find = logging_wrapper('find')
     save = logging_wrapper('save')
     remove = logging_wrapper('remove')
-    update = logging_wrapper('update', npositional=2)
-    map_reduce = logging_wrapper('map_reduce', npositional=None)
+    update = logging_wrapper('update')
+    map_reduce = logging_wrapper('map_reduce')
+    inline_map_reduce = logging_wrapper('inline_map_reduce')
 
     del logging_wrapper
+
+class DebugCursor(Cursor):
+    def __init__(self, collection_wrapper, *args, **kwargs):
+        self.collection_wrapper = collection_wrapper
+        super(DebugCursor, self).__init__(*args, **kwargs)
+
+    def _refresh(self):
+        super_meth = super(DebugCursor, self)._refresh
+        if self._Cursor__id is not None:
+            return super_meth()
+        # self.__id is None: first time the .find() iterator is entered.
+        # find() profiling happens here.
+        duration, retval = self.collection_wrapper.profile_call(super_meth)
+        kwargs = {'limit': self._Cursor__limit, 'skip': self._Cursor__skip,
+                  'sort': self._Cursor__ordering}
+        self.collection_wrapper.log('find', duration, [self._Cursor__spec], kwargs)
+        return retval

docs/source/conf.py

@@ -1,12 +1,11 @@
 # coding: utf-8
-import django_mongodb_engine
+import sys; sys.path.append('.')
+from .utils import get_current_year, get_git_head
 
 project = 'Django MongoDB Engine'
-copyright = '2011, Jonas Haag, Flavio Percoco Premoli and contributors'
-version = release = ''.join(map(str, django_mongodb_engine.__version__))
+copyright = '2010-%d, Jonas Haag, Flavio Percoco Premoli and contributors' % get_current_year()
 
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.todo',
-              'sphinx.ext.coverage']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
 
 master_doc = 'index'
 exclude_patterns = ['_build']
@@ -22,27 +21,19 @@
 
 # -- Options for HTML output ---------------------------------------------------
 
-from subprocess import check_output, CalledProcessError
-GIT_HEAD = None
-try:
-    GIT_HEAD = check_output(['git', 'rev-parse', 'HEAD'])
-except CalledProcessError:
-    pass
-except OSError, exc:
-    if exc.errno != 2:
-        raise
-
-
 html_title = project
+
 html_last_updated_fmt = '%b %d, %Y'
-if GIT_HEAD:
-    html_last_updated_fmt += ' (%s)' % GIT_HEAD[:7]
-html_show_copyright = False
+git_head = get_git_head()
+if git_head:
+    html_last_updated_fmt += ' (%s)' % git_head[:7]
+
 html_theme = 'mongodbtheme'
 html_theme_path = ['mongodbtheme', '.']
+html_show_copyright = False
 
 # Custom sidebar templates, maps document names to template names.
-html_sidebars = {'**' : 'sidebar-contribute.html'}
+html_sidebars = {'**' : ['localtoc.html', 'sidebar.html']}
 
 # If false, no module index is generated.
 html_domain_indices = False

docs/source/mongodbtheme/sidebar-contribute.html renamed to docs/source/mongodbtheme/sidebar.html

@@ -1,3 +1,9 @@
+<h3>⚠ Documentation version</h3>
+<p>
+  These are the Git version docs.
+  Docs for 0.4 (PyPI) are <a href="/0.4/">here</a>.
+</p>
+
 <h3>Help out!</h3>
 <p>
 To make this documentation even better, we'd love to receive your

docs/source/utils.py

@@ -0,0 +1,14 @@
+from datetime import datetime
+from subprocess import check_output, CalledProcessError
+
+def get_current_year():
+    return datetime.utcnow().year
+
+def get_git_head():
+    try:
+        return check_output(['git', 'rev-parse', 'HEAD'])
+    except CalledProcessError:
+        pass
+    except OSError, exc:
+        if exc.errno != 2:
+            raise