Numerous improvements and refactorings

.gitignore

@@ -1,2 +1,61 @@
-*.pyc
-/build
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+.settings
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+# Sphinx documentation
+docs/_build/
+
+# Local virtualenvs used for testing.
+/.env*
+
+# PIP install version files.
+/=*
+*.geany
+*.out

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+-   repo: https://github.com/pre-commit/mirrors-yapf
+    rev: v0.28.0
+    hooks:
+    -   id: yapf
+        args: [--in-place, --parallel, --recursive]

.style.yapf

@@ -0,0 +1,13 @@
+[style]
+BASED_ON_STYLE = pep8
+COLUMN_LIMIT = 160
+COALESCE_BRACKETS = true
+DEDENT_CLOSING_BRACKETS = true
+BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true
+SPACES_BEFORE_COMMENT = 1
+SPLIT_COMPLEX_COMPREHENSION = true
+SPACE_BETWEEN_ENDING_COMMA_AND_CLOSING_BRACKET = false
+SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT = 10
+CONTINUATION_INDENT_WIDTH = 4
+INDENT_WIDTH = 4
+CONTINUATION_ALIGN_STYLE = SPACE

.travis.yml

@@ -0,0 +1,14 @@
+# https://docs.travis-ci.com/user/reference/overview/
+dist: bionic
+sudo: required
+language: python
+python:
+-   "3.6"
+install:
+-   sudo add-apt-repository -y ppa:deadsnakes/ppa
+-   sudo apt-get -yq update
+-   sudo apt-get -yq install python3.5 python3.5-dev python3.6 python3.6-dev python3.7 python3.7-dev
+-   pip install -r pip-requirements-test.txt
+script:
+-   ./pep8.sh
+-   tox

.yapfignore

@@ -0,0 +1 @@
+database_files/migrations/*

MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include database_files/tests/fixtures *
+recursive-include database_files/tests/media *
+include pip-requirements-min-django.txt
+include pip-requirements.txt
+include pip-requirements-test.txt

README.md

@@ -1,40 +1,108 @@
-django-database-files
-=====================
+Django Database Files 3000
+==========================
 
-django-database-files is a storage system for Django that stores uploaded files 
-in the database.
+[![](https://img.shields.io/pypi/v/django-database-files-3000.svg)](https://pypi.python.org/pypi/django-database-files-3000) [![Build Status](https://img.shields.io/travis/chrisspen/django-database-files-3000.svg?branch=master)](https://travis-ci.org/chrisspen/django-database-files-3000) [![](https://pyup.io/repos/github/chrisspen/django-database-files-3000/shield.svg)](https://pyup.io/repos/github/chrisspen/django-database-files-3000)
 
-WARNING: It is generally a bad idea to serve static files from Django, 
-but there are some valid use cases. If your Django app is behind a caching 
-reverse proxy and you need to scale your application servers, it may be 
-simpler to store files in the database.
-
-Requires:
-
-  * Django 1.1
+This is a storage system for Django that stores uploaded
+files in the database. Files can be served from the database
+(usually a bad idea), the file system, or a CDN.
 
 Installation
 ------------
 
-    $ python setup.py install
+Simply install via pip with:
+
+    pip install django-database-files-3000
 
 Usage
 -----
 
-In ``settings.py``, add ``database_files`` to your ``INSTALLED_APPS`` and add this line:
+In `settings.py`, add `database_files` to your `INSTALLED_APPS` and add
+this line:
 
     DEFAULT_FILE_STORAGE = 'database_files.storage.DatabaseStorage'
 
-Although ``upload_to`` is a required argument on ``FileField``, it is not used for 
-storing files in the database. Just set it to a dummy value:
+Note, the `upload_to` parameter is still used to synchronize the files stored
+in the database with those on the file system, so new and existing fields
+should still have a value that makes sense from your base media directory.
+
+If you're using South, the initial model migrations will scan through all
+existing models for `FileFields` or `ImageFields` and will automatically
+load them into the database.
+
+If for any reason you want to re-run this bulk import task, run:
+
+    python manage.py database_files_load
+
+Additionally, if you want to export all files in the database back to the file
+system, run:
+
+    python manage.py database_files_dump
+
+Note, that when a field referencing a file is cleared, the corresponding file
+in the database and on the file system will not be automatically deleted.
+To delete all files in the database and file system not referenced by any model
+fields, run:
+
+    python manage.py database_files_cleanup
+
+Settings
+-------
+
+* `DB_FILES_AUTO_EXPORT_DB_TO_FS` = `True`|`False` (default `True`)
+
+    If true, when a file is uploaded or read from the database, a copy will be
+    exported to your media directory corresponding to the FileField's upload_to
+    path, just as it would with the default Django file storage.
+
+    If false, the file will only exist in the database.
+
+* `DATABASE_FILES_URL_METHOD` = `URL_METHOD_1`|`URL_METHOD_2` (default `URL_METHOD_1`)
+
+    Defines the method to use when rendering the web-accessible URL for a file.
+
+    If `URL_METHOD_1`, assumes all files have been exported to the filesystem and
+    uses the path corresponding to your `settings.MEDIA_URL`.
+
+    If `URL_METHOD_2`, uses the URL bound to the `database_file` view
+    to dynamically lookup and serve files from the filesystem or database.
+
+    In this case, you will also need to updates your `urls.py` to include the view
+    that serves the files:
+
+        urlpatterns = patterns('',
+            # ... the rest of your URLconf goes here ...
+
+            # Serve Database Files directly
+            url(r'', include('database_files.urls')),
+        )
+
+Development
+-----------
+
+Tests require the Python development headers to be installed, which you can install on Ubuntu with:
+
+    sudo apt-get install python-dev python3-dev python3.4-dev
+
+To run unittests across multiple Python versions, install:
+
+    sudo apt-get install python3.4-minimal python3.4-dev python3.5-minimal python3.5-dev
+
+Note, you may need to enable an [additional repository](https://launchpad.net/~fkrull/+archive/ubuntu/deadsnakes) to provide these packages.
+
+To run all [tests](http://tox.readthedocs.org/en/latest/):
 
-    upload = models.FileField(upload_to='not required')
+    export TESTNAME=; tox
 
-All your ``FileField`` and ``ImageField`` files will now be stored in the 
-database.
+To run tests for a specific environment (e.g. Python 2.7 with Django 1.4):
+
+    export TESTNAME=; tox -e py27-django15
 
-Test suite
-----------
+To run a specific test:
+
+    export TESTNAME=.test_adding_file; tox -e py27-django15
 
-    $ ./run_tests.sh
+To build and deploy a versioned package to PyPI, verify [all unittests are passing](https://travis-ci.org/chrisspen/django-database-files), and then run:
 
+    python setup.py sdist
+    python setup.py sdist upload

database_files/__init__.py

@@ -0,0 +1,2 @@
+VERSION = (1, 0, 6)
+__version__ = '.'.join(map(str, VERSION))

database_files/management/commands/database_files_cleanup.py

@@ -0,0 +1,81 @@
+from __future__ import print_function
+
+import os
+from optparse import make_option
+
+from django.conf import settings
+from django.core.files.storage import default_storage
+from django.core.management.base import BaseCommand, CommandError
+from django.db.models import FileField, ImageField, get_models
+
+from database_files.models import File
+
+
+class Command(BaseCommand):
+    args = ''
+    help = 'Deletes all files in the database that are not referenced by ' + \
+        'any model fields.'
+    option_list = BaseCommand.option_list + (
+        make_option('--dryrun',
+            action='store_true',
+            dest='dryrun',
+            default=False,
+            help='If given, only displays the names of orphaned files ' + \
+                'and does not delete them.'),
+        make_option('--filenames',
+            default='',
+            help='If given, only files with these names will be checked'),
+        )
+
+    def handle(self, *args, **options):
+        tmp_debug = settings.DEBUG
+        settings.DEBUG = False
+        names = set()
+        dryrun = options['dryrun']
+        filenames = set(_.strip() for _ in options['filenames'].split(',') if _.strip())
+        try:
+            for model in get_models():
+                print('Checking model %s...' % (model,))
+                for field in model._meta.fields:
+                    if not isinstance(field, (FileField, ImageField)):
+                        continue
+                    # Ignore records with null or empty string values.
+                    q = {'%s__isnull' % field.name: False}
+                    xq = {field.name: ''}
+                    subq = model.objects.filter(**q).exclude(**xq)
+                    subq_total = subq.count()
+                    subq_i = 0
+                    for row in subq.iterator():
+                        subq_i += 1
+                        if subq_i == 1 or not subq_i % 100:
+                            print('%i of %i' % (subq_i, subq_total))
+                        f = getattr(row, field.name)
+                        if f is None:
+                            continue
+                        if not f.name:
+                            continue
+                        names.add(f.name)
+
+            # Find all database files with names not in our list.
+            print('Finding orphaned files...')
+            orphan_files = File.objects.exclude(name__in=names)
+            if filenames:
+                orphan_files = orphan_files.filter(name__in=filenames)
+            orphan_files = orphan_files.only('name', 'size')
+            total_bytes = 0
+            orphan_total = orphan_files.count()
+            orphan_i = 0
+            print('Deleting %i orphaned files...' % (orphan_total,))
+            for f in orphan_files.iterator():
+                orphan_i += 1
+                if orphan_i == 1 or not orphan_i % 100:
+                    print('%i of %i' % (orphan_i, orphan_total))
+                total_bytes += f.size
+                if dryrun:
+                    print('File %s is orphaned.' % (f.name,))
+                else:
+                    print('Deleting orphan file %s...' % (f.name,))
+                    default_storage.delete(f.name)
+            print('%i total bytes in orphan files.' % total_bytes)
+        finally:
+            settings.DEBUG = tmp_debug

database_files/management/commands/database_files_dump.py

@@ -0,0 +1,20 @@
+import os
+from optparse import make_option
+
+from django.core.management.base import BaseCommand, CommandError
+
+from database_files.models import File
+
+
+class Command(BaseCommand):
+    option_list = BaseCommand.option_list + (
+        #        make_option('-w', '--overwrite', action='store_true',
+        #            dest='overwrite', default=False,
+        #            help='If given, overwrites any existing files.'),
+    )
+    help = 'Dumps all files in the database referenced by FileFields ' + \
+        'or ImageFields onto the filesystem in the directory specified by ' + \
+        'MEDIA_ROOT.'
+
+    def handle(self, *args, **options):
+        File.dump_files(verbose=True)