Add urlmap to repo; update check_urls;

data/index.tpl

@@ -14,7 +14,7 @@
 
 {% block body %}
 <!--META{
-    "aliases": ["/post/"]
+    "aliases": ["/post/", "/s/"]
 }-->
 
 <div class="intro">

data/urls.json

@@ -0,0 +1,171 @@
+{
+    "/": [
+        "/post/",
+        "/s/",
+        "/post",
+        "/s"
+    ],
+    "/naspeh/": [
+        "/naspeh/detail/",
+        "/naspeh/detail",
+        "/naspeh"
+    ],
+    "/post/gnome2-and-small-display/": [
+        "/naspeh/gnome-optimizaciya-okon/",
+        "/naspeh/gnome-optimizaciya-okon",
+        "/post/gnome2-and-small-display"
+    ],
+    "/post/python-code-management/": [
+        "/naspeh/python-code-management/",
+        "/naspeh/python-code-management",
+        "/post/python-code-management"
+    ],
+    "/post/python-compact-tests/": [
+        "/naspeh/lakonichnost-testov-v-python/",
+        "/naspeh/lakonichnost-testov-v-python",
+        "/post/python-compact-tests"
+    ],
+    "/post/unique-nick/": [
+        "/naspeh/unikalnyy-nik/",
+        "/naspeh/unikalnyy-nik",
+        "/post/unique-nick"
+    ],
+    "/post/zf-app-and-autoload/": [
+        "/post/avtozagruzka-klassov-v-prilozheniyah-na-zend-framework/",
+        "/blog/2008/09/25/avtozagruzka-klassov-v-prilozheniyah-na-zend-framework/",
+        "/post/avtozagruzka-klassov-v-prilozheniyah-na-zend-framework",
+        "/blog/2008/09/25/avtozagruzka-klassov-v-prilozheniyah-na-zend-framework",
+        "/post/zf-app-and-autoload"
+    ],
+    "/resume/": null,
+    "/s/2013-ru-pycon/": [
+        "/s/writing/ru-pycon-2013/",
+        "/s/writing/ru-pycon-2013",
+        "/s/2013-ru-pycon"
+    ],
+    "/s/napokaz/": null,
+    "/s/napokaz/src/": null,
+    "/trip/": null,
+    "/trip/2006-karpaty-chernogorskiy-khrebet/": [
+        "/event-63",
+        "/event-63-karpatyi-chernogorskij-hrebet",
+        "/report-66/for/event-63-karpatyi-chernogorskij-hrebet",
+        "/event-63",
+        "/event-63-karpatyi-chernogorskij-hrebet",
+        "/report-66/for/event-63-karpatyi-chernogorskij-hrebet",
+        "/trip/2006-karpaty-chernogorskiy-khrebet"
+    ],
+    "/trip/2006-karpaty-chernogorskiy-khrebet/detail/": [
+        "/report-67/for/event-63-karpatyi-chernogorskij-hrebet",
+        "/report-67/for/event-63-karpatyi-chernogorskij-hrebet",
+        "/trip/2006-karpaty-chernogorskiy-khrebet/detail"
+    ],
+    "/trip/2007-krym-tarhankut-olenevka/": [
+        "/event-12",
+        "/event-12-kryim-tarhankut-olenevka",
+        "/report-15/for/event-12-kryim-tarhankut-olenevka",
+        "/event-12",
+        "/event-12-kryim-tarhankut-olenevka",
+        "/report-15/for/event-12-kryim-tarhankut-olenevka",
+        "/trip/2007-krym-tarhankut-olenevka"
+    ],
+    "/trip/2008-krym-tarhankut-olenevka-barrakuda/": [
+        "/event-19",
+        "/event-19-kryim-tarhankut-olenevka-barrakuda",
+        "/report-20/for/event-19-kryim-tarhankut-olenevka-barrakuda",
+        "/event-19",
+        "/event-19-kryim-tarhankut-olenevka-barrakuda",
+        "/report-20/for/event-19-kryim-tarhankut-olenevka-barrakuda",
+        "/trip/2008-krym-tarhankut-olenevka-barrakuda"
+    ],
+    "/trip/2009-arabatskaya-strelka-velo/": [
+        "/event-39",
+        "/event-39-novoalekseevka-sudak-cherez-arabatskuyu-strelku",
+        "/report-47/for/event-39-novoalekseevka-sudak-cherez-arabatskuyu-strelku",
+        "/event-39",
+        "/event-39-novoalekseevka-sudak-cherez-arabatskuyu-strelku",
+        "/report-47/for/event-39-novoalekseevka-sudak-cherez-arabatskuyu-strelku",
+        "/trip/2009-arabatskaya-strelka-velo"
+    ],
+    "/trip/2009-karpaty-kosovskiy-rayon/": [
+        "/event-84",
+        "/event-84-karpatyi-kosovskij-rajon",
+        "/report-85/for/event-84-karpatyi-kosovskij-rajon",
+        "/event-84",
+        "/event-84-karpatyi-kosovskij-rajon",
+        "/report-85/for/event-84-karpatyi-kosovskij-rajon",
+        "/trip/2009-karpaty-kosovskiy-rayon"
+    ],
+    "/trip/2009-krym-novyy-svet/": [
+        "/event-32",
+        "/event-32-kryim-novyij-svet",
+        "/report-38/for/event-32-kryim-novyij-svet",
+        "/event-32",
+        "/event-32-kryim-novyij-svet",
+        "/report-38/for/event-32-kryim-novyij-svet",
+        "/trip/2009-krym-novyy-svet"
+    ],
+    "/trip/2009-krym-tarhankut-olenevka-treylery/": [
+        "/event-51",
+        "/event-51-kryim-tarhankut-olenevka-trejleryi",
+        "/report-52/for/event-51-kryim-tarhankut-olenevka-trejleryi",
+        "/event-51",
+        "/event-51-kryim-tarhankut-olenevka-trejleryi",
+        "/report-52/for/event-51-kryim-tarhankut-olenevka-trejleryi",
+        "/trip/2009-krym-tarhankut-olenevka-treylery"
+    ],
+    "/trip/2010-karpaty-svidovetskiy-khrebet/": [
+        "/event-98",
+        "/event-98-karpatyi-svidovetskij-hrebet",
+        "/report-99/for/event-98-karpatyi-svidovetskij-hrebet",
+        "/event-98",
+        "/event-98-karpatyi-svidovetskij-hrebet",
+        "/report-99/for/event-98-karpatyi-svidovetskij-hrebet",
+        "/trip/2010-karpaty-svidovetskiy-khrebet"
+    ],
+    "/trip/2010-krym-yalta-bahchisaray-velo/": [
+        "/event-107",
+        "/event-107-kryim-yalta-bahchisaraj",
+        "/report-108/for/event-107-kryim-yalta-bahchisaraj",
+        "/event-107",
+        "/event-107-kryim-yalta-bahchisaraj",
+        "/report-108/for/event-107-kryim-yalta-bahchisaraj",
+        "/trip/2010-krym-yalta-bahchisaray-velo"
+    ],
+    "/trip/2010-krym-yugo-vostochnyy-gornyy/": [
+        "/event-75",
+        "/event-75-kryim-yugo-vostochnyij-gornyij",
+        "/report-76/for/event-75-kryim-yugo-vostochnyij-gornyij",
+        "/event-75",
+        "/event-75-kryim-yugo-vostochnyij-gornyij",
+        "/report-76/for/event-75-kryim-yugo-vostochnyij-gornyij",
+        "/trip/2010-krym-yugo-vostochnyy-gornyy"
+    ],
+    "/trip/2011-karpaty-spravzhnya-kazka/": [
+        "/event-132",
+        "/event-132-karpatyi-spravzhnya-kazka",
+        "/report-133/for/event-132-karpatyi-spravzhnya-kazka",
+        "/event-132",
+        "/event-132-karpatyi-spravzhnya-kazka",
+        "/report-133/for/event-132-karpatyi-spravzhnya-kazka",
+        "/trip/2011-karpaty-spravzhnya-kazka"
+    ],
+    "/trip/2011-krym-simferopol-feodosiya-velo/": [
+        "/event-121",
+        "/event-121-kryim-velopohod-simferopol-feodosiya",
+        "/report-122/for/event-121-kryim-velopohod-simferopol-feodosiya",
+        "/event-121",
+        "/event-121-kryim-velopohod-simferopol-feodosiya",
+        "/report-122/for/event-121-kryim-velopohod-simferopol-feodosiya",
+        "/trip/2011-krym-simferopol-feodosiya-velo"
+    ],
+    "/trip/2012-karpaty-spravzhnya-kazka/": [
+        "/event-144",
+        "/event-144-karpatyi-spravzhnya-kazka",
+        "/report-145/for/event-144-karpatyi-spravzhnya-kazka",
+        "/event-144",
+        "/event-144-karpatyi-spravzhnya-kazka",
+        "/report-145/for/event-144-karpatyi-spravzhnya-kazka",
+        "/trip/2012-karpaty-spravzhnya-kazka"
+    ]
+}

manage.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 import argparse
-import re
+import json
 import subprocess
 import time
+from http.client import HTTPConnection
 from threading import Thread
-from urllib.request import urlopen
 from urllib.error import HTTPError
 
 import pusto
@@ -26,9 +26,10 @@ def sub(name, **kw):
         s.exe = lambda f: s.set_defaults(exe=f) and s
         return s
 
-    sub('test_urls', help='test urls from google')\
+    sub('test_urls', help='test url responses')\
+        .arg('-v', '--verbose', action='store_true')\
         .arg('--host', help='use host for test')\
-        .exe(lambda a: check_urls(host=a.host))
+        .exe(lambda a: check_urls(host=a.host, verbose=a.verbose))
 
     sub('deploy', help='deploy to server')\
         .exe(lambda a: ssh(
@@ -48,58 +49,50 @@ def sub(name, **kw):
         args.exe(args)
 
 
-def check_urls(host=None):
+def check_urls(host=None, verbose=False):
+    log = lambda *a: verbose and print(*a)
+
     if not host:
         pusto.process('build')
-        host = 'http://localhost:9000'
+        host = 'localhost:9000'
         args = 'run --port=9000 --no-reloader'.split(' ')
         server = Thread(target=pusto.process, args=args)
         server.daemon = True
         server.start()
         time.sleep(2)
 
-    urls = [
-        '/resume',
-        '/naspeh/detail/',
-        '/naspeh/unikalnyy-nik/',
-        '/naspeh/gnome-optimizaciya-okon/',
-        '/naspeh/python-code-management/',
-        '/naspeh/lakonichnost-testov-v-python/',
-        '/post/avtozagruzka-klassov-v-prilozheniyah-na-zend-framework',
-        (
-            '/blog/2008/09/25/'
-            'avtozagruzka-klassov-v-prilozheniyah-na-zend-framework/'
-        ),
-        '/yandex_5ad3ffab17496674.txt',
-        '/googlee71e35f8e9cbd607.html',
-        '/s/napokaz/',
-        '/s/writing/ru-pycon-2013/',
-    ]
-
-    def get(url):
+    with open('data/urls.json', 'br') as f:
+        urls = json.loads(f.read().decode())
+
+    def get(url, expected_code=200):
         comment = ''
         try:
-            res = urlopen(host + url)
-            text = res.read().decode()
-            new = re.search(r'http-equiv="refresh" content="0;url=(.*)"', text)
-            if new:
-                new_url = new.groups(1)[0]
-                res = urlopen(host + new_url)
-                comment = 'Manual redirect'
+            conn = HTTPConnection(host)
+            conn.request('HEAD', url)
+            res = conn.getresponse()
             code = res.status
-            res_url = res.url
+            if code == 200:
+                res_url = url
+            else:
+                res_url = '/' + res.info().get('Location', '')
+                res_url = res_url.lstrip('http://' + host)
         except HTTPError as e:
             code = e.code
             res_url = None
-        return code, res_url, comment
 
-    err = []
-    for url in urls:
-        code, res_url, comment = get(url)
-        print('%s %s %s # %s' % (url, code, res_url, comment))
-        expected_code = 200
+        err = []
         if code != expected_code:
-            err += ['%s (%r != %r)' % (url, code, expected_code)]
+            err = ['%s (%r != %r)' % (url, code, expected_code)]
+        else:
+            log('%s %s %s # %s' % (url, code, res_url, comment))
+        return err
+
+    err = []
+    for url in sorted(urls.keys()):
+        aliases = urls.get(url)
+        err += get(url)
+        for alias in aliases or []:
+            err += get(alias, expected_code=301)
     if err:
         print('Errors:')
         print('\n'.join(err))

pusto.py

@@ -29,8 +29,10 @@
 INDEX_FILES = ['index.' + t for t in 'py html tpl rst md'.split(' ')]
 
 
-def get_urls(src_dir):
-    pages = get_pages(src_dir)
+def get_urls(src_dir, pages=None):
+    if pages is None:
+        pages = get_pages(src_dir)
+
     urls = []
     for url, page in pages.items():
         if page.html:
@@ -235,8 +237,10 @@ def build(src_dir, build_dir, nginx_file=None):
         shutil.rmtree(build_dir)
     shutil.copytree(src_dir, build_dir)
 
-    urls = get_urls(build_dir)
+    pages = get_pages(build_dir)
+    urls = get_urls(build_dir, pages=pages)
     nginx = {}
+    urlmap = {}
     for url, page in urls:
         if url != page.url and (url + '/') != page.url:
             nginx[url.rstrip('/')] = page.url
@@ -254,6 +258,19 @@ def build(src_dir, build_dir, nginx_file=None):
 
         with open(nginx_file, 'bw') as f:
             f.write(lines.encode())
+
+    urlmap = dict(
+        [url, page.aliases]
+        for url, page in pages.items()
+        if page.index_file
+    )
+    urlmap_file = os.path.join(src_dir, 'urls.json')
+    with open(urlmap_file, 'bw') as f:
+        urlmap = json.dumps(
+            urlmap, sort_keys=True, indent=4, separators=(',', ': ')
+        )
+        f.write(urlmap.encode())
+
     print(' * Build successful')
     return urls