Merge pull request #2 from joshowen/2.0

Updated for modern Django versions

Author: voronind

.gitignore

@@ -0,0 +1,6 @@
+
+.idea/
+
+*.pyc
+
+.tox/

.travis.yml

@@ -0,0 +1,19 @@
+language: python
+python:
+    - "3.5"
+    - "3.4"
+    - "2.7"
+sudo: false
+env:
+    - TOX_ENV=django19
+    - TOX_ENV=django18
+    - TOX_ENV=django110
+    - TOX_ENV=djangomaster
+matrix:
+    fast_finish: true
+    allow_failures:
+      - env: TOX_ENV=djangomaster
+install:
+    - pip install tox
+script:
+    - tox -e $TOX_ENV

CHANGELOG

@@ -1,3 +1,6 @@
+2.0   - TBD
+=====================
+    - Support Django-1.8+ and Python3
 
 1.1   - December 2012
 =====================

README.rst

@@ -2,7 +2,7 @@
 django-url-robots
 =========================
 
-``Django`` ``robots.txt`` generator. Based on using decorated ``django.conf.urls.defaults.url``.
+``Django`` ``robots.txt`` generator. Based on using decorated ``django.conf.urls.url``.
 It gets ``urlpatterns`` and replaces ambiguous parts by ``*``.
 
 Installation & Usage
@@ -24,28 +24,28 @@ The recommended way to install django-url-robots is with `pip <http://pypi.pytho
 
 3. Add url_robots view to your root URLconf::
 
-    urlpatterns += patterns('',
-        url(r'^robots\.txt$', 'url_robots.views.robots_txt'),
-        )
+    urlpatterns += [
+        url(r'^robots\.txt$', url_robots.views.robots_txt),
+        ]
 
-4. Describe rules by boolean keyword argument ``robots_allow`` using for it ``url_robots.utils.url`` instead ``django.conf.urls.defaults.url``::
+4. Describe rules by boolean keyword argument ``robots_allow`` using for it ``url_robots.utils.url`` instead ``django.conf.urls.url``::
 
     from url_robots.utils import url
     
-    urlpatterns += patterns('',
-       url('^profile/private$', 'view', robots_allow=False),
-       )
+    urlpatterns += [
+       url('^profile/private$', views.some_view, robots_allow=False),
+       ]
  
-``django-url-robots`` tested with ``Django-1.3``. Encodes unicode characters by percent-encoding.
+``django-url-robots`` tested with ``Django-1.8+``. Encodes unicode characters by percent-encoding.
 
 Settings
 ====================
 
 In this moment there are only one option to define template of ``robots.txt`` file::
 
-    urlpatterns += patterns('',
-        url(r'^robots\.txt$', 'url_robots.views.robots_txt', {'template': 'my_awesome_robots_template.txt'}),
-        )
+    urlpatterns += [
+        url(r'^robots\.txt$', url_robots.views.robots_txt, {'template': 'my_awesome_robots_template.txt'}),
+        ]
 
 Example
 ===================
@@ -57,23 +57,22 @@ robots_template.txt::
 
 urls.py::
 
-    from django.conf.urls.defaults import patterns, include
+    from django.conf.urls import include
 
-    urlpatterns = patterns('',
+    urlpatterns = [
         url(r'^profile', include('url_robots.tests.urls_profile')),
-    )
+    ]
 
 urls_profile.py::
 
-    from django.conf.urls.defaults import patterns
     from url_robots.utils import url
 
-    urlpatterns = patterns('',
-        url(r'^s$', 'view', name='profiles', robots_allow=True),
-        url(r'^/(?P<nick>\w+)$', 'view'),
-        url(r'^/(?P<nick>\w+)/private', 'view', name='profile_private', robots_allow=False),
-        url(r'^/(?P<nick>\w+)/public', 'view', name='profile_public', robots_allow=True),
-        )
+    urlpatterns = [
+        url(r'^s$', views.some_view, name='profiles', robots_allow=True),
+        url(r'^/(?P<nick>\w+)$', views.some_view),
+        url(r'^/(?P<nick>\w+)/private', views.some_view, name='profile_private', robots_allow=False),
+        url(r'^/(?P<nick>\w+)/public', views.some_view, name='profile_public', robots_allow=True),
+        ]
 
 Resulting robots.txt::
 

requirements.txt

@@ -0,0 +1 @@
+Django>=1.6

runtests.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+import sys
+
+
+DEFAULT_SETTINGS = dict(
+    INSTALLED_APPS=[
+        'django.contrib.auth',
+        'django.contrib.contenttypes',
+        'django.contrib.sites',
+        'url_robots',
+    ],
+    DATABASES={
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3'
+        }
+    },
+    ROOT_URLCONF='tests.test_utils.urls',
+    SITE_ID=1,
+    MIDDLEWARE_CLASSES=[
+        'django.middleware.http.ConditionalGetMiddleware',
+        'django.contrib.sessions.middleware.SessionMiddleware',
+        'django.contrib.auth.middleware.AuthenticationMiddleware',
+        'django.contrib.messages.middleware.MessageMiddleware',
+        'django.middleware.csrf.CsrfViewMiddleware',
+        'django.middleware.locale.LocaleMiddleware',
+        'django.middleware.common.CommonMiddleware',
+    ],
+)
+
+
+def runtests():
+    import django
+    from django.conf import settings
+
+    # Compatibility with Django 1.7's stricter initialization
+    if not settings.configured:
+        settings.configure(**DEFAULT_SETTINGS)
+    if hasattr(django, 'setup'):
+        django.setup()
+
+    from django.test.runner import DiscoverRunner
+    test_args = ['url_robots.tests']
+    failures = DiscoverRunner(
+            verbosity=1, interactive=True, failfast=False
+    ).run_tests(test_args)
+    sys.exit(failures)
+
+
+if __name__ == '__main__':
+    runtests()

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name='django-url-robots',
-    version='1.1',
+    version='2.0',
     description='Django robots.txt generator',
     long_description=long_description,
     url='http://github.com/dimka665/django-url-robots',
@@ -24,4 +24,4 @@
         'Environment :: Web Environment',
         'Framework :: Django',
         ],
-)
+)

tox.ini

@@ -0,0 +1,15 @@
+[tox]
+envlist =
+       django18,
+       django19,
+       django110,
+       django{master}
+
+[testenv]
+deps =
+        django18: Django==1.8.16
+        django19: Django==1.9.11
+        django110: Django==1.10.3
+        djangomaster: https://github.com/django/django/archive/master.tar.gz
+
+commands = {env:COMMAND:python} runtests.py

url_robots/templates/robots.txt

@@ -0,0 +1,3 @@
+User-agent: *
+{{ rules|safe }}
+Crawl-delay: 1

url_robots/templates/robots_template.txt

@@ -1,7 +1,6 @@
 # coding=utf-8
 
-# todo On Django 1.4 change testcase to django.test.SimpleTestCase
-from django.utils import unittest
+import unittest
 
 from url_robots.utils import clean_pattern, join_patterns, create_rules
 
@@ -46,8 +45,8 @@ def test_begin_of_string(self):
         self.clean_equal(r'^ab', 'ab')
 
     def test_end_of_string(self):
-        self.assertEqual(clean_pattern(ur'ab$'), u'/ab$')
-        self.assertEqual(clean_pattern(ur'ab.$'), u'/ab*')
+        self.assertEqual(clean_pattern(r'ab$'), u'/ab$')
+        self.assertEqual(clean_pattern(r'ab.$'), u'/ab*')
 
     def test_greedy_repetitions(self):
         self.clean_equal(r'a.*b', 'a*b')
@@ -98,7 +97,7 @@ def test_urlquote(self):
         self.clean_equal(r'star-\*', 'star-%2A')
         self.clean_equal(r'dollar-\$', 'dollar-%24')
         # some russian language
-        self.clean_equal(ur'path/в никуда/', ur'path/%D0%B2%20%D0%BD%D0%B8%D0%BA%D1%83%D0%B4%D0%B0/')
+        self.clean_equal(u'path/в никуда/', r'path/%D0%B2%20%D0%BD%D0%B8%D0%BA%D1%83%D0%B4%D0%B0/')
 
 
 class JoinPatternTestCase(unittest.TestCase):
@@ -137,9 +136,9 @@ def test_1(self):
 
 class CreateRulesTestCase(unittest.TestCase):
     def setUp(self):
-        self.expected_rules = '''Allow: /profiles$                                                                                                        # /profiles$                             name=profiles
-Disallow: /profile/*/private*                                                                                            # /profile/*/private*                    name=profile_private
-Allow: /profile/*/public*                                                                                                # /profile/*/public*                     name=profile_public'''
+        self.expected_rules = '''Allow: /profiles$
+Disallow: /profile/*/private*
+Allow: /profile/*/public*'''
 
     def test_create_rules_for_profiles(self):
         rules = create_rules('url_robots.tests.urls')

url_robots/tests/tests.py

@@ -1,10 +1,10 @@
 #coding=utf-8
 
-from django.conf.urls.defaults import patterns, include
+from django.conf.urls import include
 
 from url_robots.utils import url
 
 
-urlpatterns = patterns('',
+urlpatterns = [
     url(r'^profile', include('url_robots.tests.urls_profile')),
-)
+]

url_robots/tests/urls.py

@@ -1,12 +1,11 @@
 #coding=utf-8
 
-from django.conf.urls.defaults import patterns
 
 from url_robots.utils import url
 
-urlpatterns = patterns('',
-    url(r'^s$', 'view', name='profiles', robots_allow=True),
-    url(r'^/(?P<nick>\w+)$', 'view'),
-    url(r'^/(?P<nick>\w+)/private', 'view', name='profile_private', robots_allow=False),
-    url(r'^/(?P<nick>\w+)/public', 'view', name='profile_public', robots_allow=True),
-    )
+urlpatterns = [
+    url(r'^s$', lambda x: (), name='profiles', robots_allow=True),
+    url(r'^/(?P<nick>\w+)$', lambda x: ()),
+    url(r'^/(?P<nick>\w+)/private', lambda x: (), name='profile_private', robots_allow=False),
+    url(r'^/(?P<nick>\w+)/public', lambda x: (), name='profile_public', robots_allow=True),
+    ]

url_robots/tests/urls_profile.py

@@ -2,16 +2,24 @@
 import sre_parse
 from sre_constants import LITERAL, AT, AT_BEGINNING, AT_END
 
-from urllib import quote, unquote
-
-from django.conf.urls.defaults import url
+try:
+    # Python 3
+    from urllib.parse import quote, unquote
+    unichr = chr
+except ImportError:
+    # Python 2
+    from urllib import quote, unquote
+
+from django.conf.urls import url
 from django.core.urlresolvers import get_urlconf, get_resolver, RegexURLResolver
 
-# decorator for django.conf.urls.defaults.url
+
 def robots_decorator(url_function):
-    def url_extended(regex, view, kwargs=None, name=None, prefix='',
-                     robots_allow=None):
-        resolver_or_pattern = url_function(regex, view, kwargs=kwargs, name=name, prefix=prefix)
+    """
+    Decorator for django.conf.urls.url
+    """
+    def url_extended(regex, view, kwargs=None, name=None, robots_allow=None):
+        resolver_or_pattern = url_function(regex, view, kwargs=kwargs, name=name)
 
         resolver_or_pattern.robots_allow = robots_allow
         return resolver_or_pattern
@@ -22,15 +30,22 @@ def url_extended(regex, view, kwargs=None, name=None, prefix='',
 
 
 def create_rules(urlconf=None):
+    """
+    Creates rules from conf
+    """
     if urlconf is None:
         urlconf = get_urlconf()
+
     root_resolver = get_resolver(urlconf)
     rule_list = create_rule_list(root_resolver, '')
+
     return u'\n'.join(rule_list)
 
 
 def create_rule_list(parent_resolver, abs_pattern):
-
+    """
+    Creates usable rule list
+    """
     rule_list = []
 
     for resolver in parent_resolver.url_patterns:
@@ -49,15 +64,6 @@ def create_rule_list(parent_resolver, abs_pattern):
         if rule:
             path = clean_pattern(pattern)
             rule += path
-            length = 120
-            rule = rule.ljust(length)
-
-            rule += ' # ' + unquote(path).decode('utf8')
-            length += 40
-            rule = rule.ljust(length)
-
-            rule += '  name=' + (getattr(resolver, 'name', '') or '')
-
             rule_list.append(rule)
 
         if isinstance(resolver, RegexURLResolver):
@@ -67,6 +73,9 @@ def create_rule_list(parent_resolver, abs_pattern):
 
 
 def join_patterns(pattern1, pattern2):
+    """
+    Joins URL patterns
+    """
     if pattern1.endswith('$'):
         return pattern1
 
@@ -79,10 +88,13 @@ def join_patterns(pattern1, pattern2):
     return pattern1
 
 
-# pattern => token
-# '2'     => ('literal', 50)
-# '2|3'   => ('in', [('literal', 50), ('literal', 51)])
 def clean_pattern(pattern):
+    """
+    Cleans URL patterns
+     * pattern => token
+     * '2'     => ('literal', 50)
+     * '2|3'   => ('in', [('literal', 50), ('literal', 51)])
+    """
     star = '*'
     parsed = sre_parse.parse(pattern)
     literals = []
@@ -91,7 +103,6 @@ def clean_pattern(pattern):
         if token[0] == LITERAL:
             character = quote(unichr(token[1]).encode('utf8'))
             literals.append(character)
-
         elif token[0] == AT:
             pass