Merge remote-tracking branch 'frodrigo/master'

* frodrigo/master:
  Introduce langue variants #718 #811
  py3: Remove unicode compatibility
  Fix fix filter on allow_fix_override #824

Author: jocelynj

analysers/analyser_osmosis_soundex.py

@@ -258,7 +258,7 @@ def analyser_osmosis_common(self):
         if not self.scripts:
             return
 
-        if "language" in self.config.options and self.config.options["language"] == "fr":
+        if "language" in self.config.options and self.config.options["language"].startswith("fr"):
             self.run(sql01_fr)
             self.run(sql03.format("fn_soundex2"))
         else:

analysers/analyser_sax.py

@@ -28,12 +28,6 @@
 from io import open  # In python3 only, this import is not required
 from modules import OsmoseLog
 
-try:
-    unicode # Python 2
-except:
-    unicode = str
-
-###########################################################################
 
 class Analyser_Sax(Analyser):
 
@@ -195,6 +189,7 @@ def NodeCreate(self, data):
                     subclass = e.get("subclass", 0)
                     text = e.get("text", {})
                     fix = e.get("fix")
+                    allow_fix_override = e.get('allow_fix_override')
 
                     self.error_file.error(
                         classs,
@@ -203,7 +198,8 @@ def NodeCreate(self, data):
                         [data["id"]],
                         ["node"],
                         fix,
-                        {"position": [data], "node": [data]})
+                        {"position": [data], "node": [data]},
+                        allow_override = allow_fix_override)
                 except:
                     self._err("Error on error %s from %s" % (str(e), str(err)))
                     raise
@@ -349,6 +345,7 @@ def RelationCreate(self, data):
                     subclass = e.get("subclass", 0)
                     text = e.get("text", {})
                     fix = e.get("fix")
+                    allow_fix_override = e.get('allow_fix_override')
 
                     self.error_file.error(
                         classs,
@@ -357,7 +354,8 @@ def RelationCreate(self, data):
                         [data["id"]],
                         ["relation"],
                         fix,
-                        {"position": [node], "relation": [data]})
+                        {"position": [node], "relation": [data]},
+                        allow_override = allow_fix_override)
                 except:
                     self._err("Error on error %s from %s" % (str(e), str(err)))
                     raise
@@ -433,7 +431,7 @@ def _load_plugins(self):
         conf_limit = set()
         for i in ("country", "language"):
             if i in self.config.options:
-                if isinstance(self.config.options[i], str) or isinstance(self.config.options[i], unicode):
+                if isinstance(self.config.options[i], str):
                     conf_limit.add(self.config.options[i])
 
         # load plugins

mapcss/mapcss2osmose.py

@@ -787,7 +787,6 @@ def main(_, mapcss):
     asserts = build_tests(tests)
 
     mapcss = ("""#-*- coding: utf-8 -*-
-from __future__ import unicode_literals
 import modules.mapcss_lib as mapcss
 import regex as re
 

modules/DictCursorUnicode.py

@@ -25,19 +25,10 @@
 
 class DictRowUnicode50(psycopg2.extras.DictRow):
     def __getitem__(self, x):
-        if sys.version_info < (3, ):
-            # Hack to make DictRow support of unicode key in Pyhton 2
-            if isinstance(x, unicode):
-                return super(DictRowUnicode50, self).__getitem__(x[:50].encode('utf-8'))
-            elif isinstance(x, str):
-                return super(DictRowUnicode50, self).__getitem__(x.decode('utf-8')[:50].encode('utf-8'))
-            else:
-                return super(DictRowUnicode50, self).__getitem__(x)
+        if isinstance(x, str):
+            return super(DictRowUnicode50, self).__getitem__(x[:50])
         else:
-            if isinstance(x, str):
-                return super(DictRowUnicode50, self).__getitem__(x[:50])
-            else:
-                return super(DictRowUnicode50, self).__getitem__(x)
+            return super(DictRowUnicode50, self).__getitem__(x)
 
 class DictCursorUnicode50(psycopg2.extras.DictCursor):
     def __init__(self, *args, **kwargs):

modules/SourceVersion.py

@@ -23,15 +23,11 @@
 import time, os
 import hashlib
 
-try:
-    unicode # Python 2
-except:
-    unicode = str
 
 def version(*sources):
     h = hashlib.md5()
     for source in sources:
-        if (isinstance(source, str) or isinstance(source, unicode)) and os.path.exists(source):
+        if isinstance(source, str) and os.path.exists(source):
             h.update(open(source, 'rb').read())
         elif isinstance(source, int):
             h.update(str(source).encode('utf-8'))

modules/downloader.py

@@ -29,10 +29,6 @@
 from datetime import datetime
 from . import config
 
-try:
-    unicode
-except:
-    unicode = str
 
 HTTP_DATE_FMT = "%a, %d %b %Y %H:%M:%S GMT"
 
@@ -226,15 +222,15 @@ def test_urlopen(self):
     def test_urlread(self):
         dst = urlread(self.url, 10)
         assert dst
-        self.assertIsInstance(dst, unicode)
+        self.assertIsInstance(dst, str)
         self.check_content(dst)
         exp_content = open(update_cache(self.url, 100), "r", encoding="utf-8").read()
         self.assertEqual(dst, exp_content, "urlread doesn't give expected result")
 
     def test_urlread_fr(self):
         dst = urlread(self.url_fr, 10)
         assert dst
-        self.assertIsInstance(dst, unicode)
+        self.assertIsInstance(dst, str)
         self.check_content(dst)
         exp_content = open(update_cache(self.url_fr, 100), "r", encoding="utf-8").read()
         self.assertEqual(dst, exp_content, "urlread doesn't give expected result")

modules/languages.py

@@ -45,6 +45,9 @@
   'fi': ['Latin'],
   'fo': ['Latin'],
   'fr': [u'[A-Za-zÉÀÈÙÂÊÎÔÛËÏÜŸÇŒÆéàèùâêîôûëïüÿçœæ]'],
+  'fr_GF': [u'[A-Za-zÉÀÈÙÂÊÎÔÛËÏÜŸÇŒÆÃĨÕŨÖƗéàèùâêîôûëïüÿçœæãĩõũöɨ]'],
+  'fr_LU': [u'[A-Za-zÉÀÈÙÂÊÎÔÛÄËÏÜŸÇŒÆéàèùâêîôûäëïüÿçœæ]'],
+  'fr_PF': [u'[A-Za-zÉÀÈÙÂÊÎÔÛËÏÜŸÇŒÆĀĪŌŪéàèùâêîôûëïüÿçœæāīōū]'],
   'fy': ['Latin'],
   'ga': ['Latin'],
   'gl': ['Latin'],
@@ -110,7 +113,11 @@ def scripts(languages):
         languages = [languages]
     all_scripts = []
     for language in languages:
-        scripts = language2scripts[language]
+        if language in language2scripts:
+            scripts = language2scripts[language]
+        elif '_' in language:
+            l = language.split('_')[0]
+            scripts = language2scripts[l]
         if not scripts:
             return
         all_scripts += scripts

modules/oh_sanitizer/main.py

@@ -8,11 +8,6 @@
 import lark as _lark
 from lark.lexer import Token as _Token
 
-try:
-    unicode # Python 2
-except:
-    unicode = str
-
 
 def get_parser():
     """
@@ -373,7 +368,7 @@ def sanitize_field(field):
     """
     # The exceptions catching should be modified when the next version
     # 'lark-parser' will be released (it introduce a better exception handling).
-    if not isinstance(field, str) and not isinstance(field, unicode):
+    if not isinstance(field, str):
         raise TypeError("The field must be a string.")
     if _re.match("[0-9]{4} [0-9].+", field):
         raise SanitizeError("This field can not be parsed properly.")

osmose_config.py

@@ -506,7 +506,7 @@ class gen(default_country):
 ], **{'addr:city-admin_level': '8,9'})
 
 france_departement_dom("guadeloupe", 1401835, "FR-GP", dep_code=971, proj=32620, phone_code="590")
-france_departement_dom("guyane", 1260551, "FR-GF", dep_code=973, proj=2972, phone_code="594")
+france_departement_dom("guyane", 1260551, "FR-GF", dep_code=973, language='fr_GF', proj=2972, phone_code="594")
 france_departement_dom("martinique", 1891495, "FR-MQ", dep_code=972, proj=32620, phone_code="596")
 france_departement_dom("mayotte", 1259885, "FR-YT", dep_code=976, proj=32738, phone_code="262")
 france_departement_dom("reunion", 1785276, "FR-RE", dep_code=974, proj=2975, phone_code="262")
@@ -527,7 +527,7 @@ class gen(default_country):
 france_com(["central-america", "saint_martin"], 1891583, "FR-MF", proj=2969, phone_code="590", country="saintmartin")
 france_com(["north-america", "saint_pierre_et_miquelon"], 233377, "FR-PM", proj=32621, phone_code="508", country="saintpierreetmiquelon")
 france_com(["south-america", "wallis_et_futuna"], 290162, "FR-WF", proj=32701, phone_code="681", country="wallisetfutuna")
-france_com(["south-america", "polynesie"], 3412620, "FR-PF", proj=32706, phone_code="689", phone_len=9)
+france_com(["south-america", "polynesie"], 3412620, "FR-PF", language='fr_PF', proj=32706, phone_code="689", phone_len=9)
 france_com(["australia-oceania", "new-caledonia"], 3407643, "NC", download_repo=GEOFABRIK, proj=3163, country="nouvellecaledonie",
     phone_code="687", phone_len=6, phone_format=r"^[+]%s([- ./]*[0-9]){5}[0-9]$", phone_international='00')
 
@@ -600,7 +600,7 @@ class gen(default_country):
 del(lithuania.analyser["osmosis_highway_cul-de-sac_level"]) # follow official highway classification
 del(lithuania.analyser["osmosis_highway_broken_level_continuity"]) # follow official highway classification
 default_country("europe", "latvia", 72594, {"country": "LV","language": "lv", "proj": 32634}, download_repo=GEOFABRIK)
-default_country("europe", "luxembourg", 2171347, {"country": "LU", "language": "lb", "proj": 2169, "boundary_detail_level": 6})
+default_country("europe", "luxembourg", 2171347, {"country": "LU", "language": "fr_LU", "proj": 2169, "boundary_detail_level": 6})
 default_country("europe", "malta", 365307, {"country": "MT", "language": "en", "driving_side": "left", "proj": 32633})
 default_country("europe", "macedonia", 53293, {"country": "MK", "language": "sq", "proj": 32634})
 default_country("europe", "moldova", 58974, {"country": "MD", "language": "ro", "proj": 32635}, download_repo=GEOFABRIK)

plugins/Bicycle.py

@@ -1,5 +1,4 @@
 #-*- coding: utf-8 -*-
-from __future__ import unicode_literals
 import modules.mapcss_lib as mapcss
 import regex as re