Merge remote-tracking branch 'origin/master' into type_dsl

Author: mr-c

schema_salad/makedoc.py

@@ -383,7 +383,7 @@ def extendsfrom(item, ex):
 
 
 def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
-    # type: (List[Dict[str, Any]], IO[Any], str, Dict, str, str) -> None
+    # type: (List[Dict[unicode, Any]], IO[Any], str, Dict, str, str) -> None
     toc = ToC()
     toc.start_numbering = False
 
@@ -475,7 +475,7 @@ def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
 
     args = parser.parse_args()
 
-    s = []  # type: List[Dict[str, Any]]
+    s = []  # type: List[Dict[unicode, Any]]
     a = args.schema
     with open(a) as f:
         if a.endswith("md"):

schema_salad/ref_resolver.py

@@ -105,13 +105,13 @@ def __init__(self, ctx, schemagraph=None, foreign_properties=None,
         self.vocab = {}  # type: Dict[unicode, Any]
         self.rvocab = {}  # type: Dict[unicode, Any]
         self.idmap = None  # type: Dict[unicode, Any]
-        self.mapPredicate = None  # type: Dict[unicode, Any]
+        self.mapPredicate = None  # type: Dict[unicode, unicode]
         self.type_dsl_fields = None # type: Set[str]
 
         self.add_context(ctx)
 
     def expand_url(self, url, base_url, scoped_id=False, vocab_term=False, scoped_ref=None):
-        # type: (Union[str, unicode], Union[str, unicode], bool, bool) -> Union[str, unicode]
+        # type: (Union[str, unicode], Union[str, unicode], bool, bool, bool) -> Union[str, unicode]
         if url in ("@id", "@type"):
             return url
 
@@ -249,10 +249,10 @@ def add_context(self, newcontext, baseuri=""):
         _logger.debug("vocab is %s", self.vocab)
 
     def resolve_ref(self, ref, base_url=None, checklinks=True):
-        # type: (Union[Dict[str, Any], str, unicode], Union[str, unicode], bool) -> Tuple[Union[Dict[str, Any], str, unicode], Dict[str, Any]]
+        # type: (Union[Dict[unicode, Any], str, unicode], Union[str, unicode], bool) -> Tuple[Union[Dict[unicode, Any], str, unicode], Dict[str, Any]]
         base_url = base_url or 'file://%s/' % os.path.abspath('.')
 
-        obj = None  # type: Dict[str, Any]
+        obj = None  # type: Dict[unicode, Any]
         inc = False
 
         # If `ref` is a dict, look for special directives.
@@ -333,24 +333,31 @@ def resolve_ref(self, ref, base_url=None, checklinks=True):
 
 
     def _resolve_idmap(self, document, loader):
+        # type: (Dict[unicode, Union[Dict[unicode, Dict[unicode, unicode]], List[Dict[unicode, Any]]]], Loader) -> None
         # Convert fields with mapSubject into lists
         # use mapPredicate if the mapped value isn't a dict.
         for idmapField in loader.idmap:
-            if (idmapField in document and isinstance(document[idmapField], dict) and
-                "$import" not in document[idmapField] and
-                "$include" not in document[idmapField]):
-                ls = []
-                for k in sorted(document[idmapField].keys()):
-                    v = document[idmapField][k]
-                    if not isinstance(v, dict):
-                        if idmapField in loader.mapPredicate:
-                            v = {loader.mapPredicate[idmapField]: v}
+            if (idmapField in document):
+                idmapFieldValue = document[idmapField]
+                if (isinstance(idmapFieldValue, dict)
+                        and "$import" not in idmapFieldValue
+                        and "$include" not in idmapFieldValue):
+                    ls = []
+                    for k in sorted(idmapFieldValue.keys()):
+                        val = idmapFieldValue[k]
+                        v = None  # type: Dict[unicode, Any]
+                        if not isinstance(val, dict):
+                            if idmapField in loader.mapPredicate:
+                                v = {loader.mapPredicate[idmapField]: val}
+                            else:
+                                raise validate.ValidationException(
+                                    "mapSubject '%s' value '%s' is not a dict"
+                                    "and does not have a mapPredicate", k, v)
                         else:
-                            raise validate.ValidationException(
-                                "mapSubject '%s' value '%s' is not a dict and does not have a mapPredicate", k, v)
-                    v[loader.idmap[idmapField]] = k
-                    ls.append(v)
-                document[idmapField] = ls
+                            v = val
+                        v[loader.idmap[idmapField]] = k
+                        ls.append(v)
+                    document[idmapField] = ls
 
     typeDSLregex = re.compile(r"^([^[?]+)(\[\])?(\?)?$")
 
@@ -389,21 +396,26 @@ def _resolve_type_dsl(self, document, loader):
                     document[d] = uniq
 
     def _resolve_identifier(self, document, loader, base_url):
+        # type: (Dict[unicode, unicode], Loader, unicode) -> unicode
         # Expand identifier field (usually 'id') to resolve scope
         for identifer in loader.identifiers:
             if identifer in document:
                 if isinstance(document[identifer], basestring):
                     document[identifer] = loader.expand_url(
-                        document[identifer], base_url, scoped_id=True)
-                    if document[identifer] not in loader.idx or isinstance(loader.idx[document[identifer]], basestring):
+                        document[identifer], base_url, scoped=True)
+                    if (document[identifer] not in loader.idx
+                            or isinstance(
+                                loader.idx[document[identifer]], basestring)):
                         loader.idx[document[identifer]] = document
                     base_url = document[identifer]
                 else:
                     raise validate.ValidationException(
-                        "identifier field '%s' must be a string" % (document[identifer]))
+                        "identifier field '%s' must be a string"
+                        % (document[identifer]))
         return base_url
 
     def _resolve_identity(self, document, loader, base_url):
+        # type: (Dict[unicode, List[unicode]], Loader, unicode) -> None
         # Resolve scope for identity fields (fields where the value is the
         # identity of a standalone node, such as enum symbols)
         for identifer in loader.identity_links:
@@ -417,6 +429,7 @@ def _resolve_identity(self, document, loader, base_url):
                                 n]] = document[identifer][n]
 
     def _normalize_fields(self, document, loader):
+        # type: (Dict[unicode, unicode], Loader) -> None
         # Normalize fields which are prefixed or full URIn to vocabulary terms
         for d in document:
             d2 = loader.expand_url(d, "", scoped_id=False, vocab_term=True)
@@ -425,26 +438,28 @@ def _normalize_fields(self, document, loader):
                 del document[d]
 
     def _resolve_uris(self, document, loader, base_url):
+        # type: (Dict[unicode, Union[unicode, List[unicode]]], Loader, unicode) -> None
         # Resolve remaining URLs based on document base
         for d in loader.url_fields:
             if d in document:
-                if isinstance(document[d], basestring):
+                datum = document[d]
+                if isinstance(datum, (str, unicode)):
                     document[d] = loader.expand_url(
-                        document[d], base_url, scoped_id=False,
+                        datum, base_url, scoped_id=False,
                         vocab_term=(d in loader.vocab_fields),
                         scoped_ref=self.scoped_ref_fields.get(d))
-                elif isinstance(document[d], list):
+                elif isinstance(datum, list):
                     document[d] = [
                         loader.expand_url(
                             url, base_url, scoped_id=False,
                             vocab_term=(d in loader.vocab_fields),
                             scoped_ref=self.scoped_ref_fields.get(d))
                         if isinstance(url, (str, unicode))
-                        else url for url in document[d]]
+                        else url for url in datum]
 
 
     def resolve_all(self, document, base_url, file_base=None, checklinks=True):
-        # type: (Any, Union[str, unicode], Union[str, unicode], bool) -> Tuple[Any, Dict[str, Any]]
+        # type: (Union[Dict[unicode, Any], List[Union[unicode, Dict]]], Union[str, unicode], Union[str, unicode], bool) -> Tuple[Any, Dict[str, Any]]
         loader = self
         metadata = {}  # type: Dict[str, Any]
         if file_base is None:
@@ -515,7 +530,7 @@ def resolve_all(self, document, base_url, file_base=None, checklinks=True):
                     val = document[i]
                     if isinstance(val, dict) and "$import" in val:
                         l, _ = loader.resolve_ref(val, base_url=file_base, checklinks=False)
-                        if isinstance(l, list):
+                        if isinstance(l, list):  # never true?
                             del document[i]
                             for item in aslist(l):
                                 document.insert(i, item)

schema_salad/schema.py

@@ -168,7 +168,7 @@ def get_metaschema():
 
 
 def load_schema(schema_ref, cache=None):
-    # type: (Union[unicode, Dict[str, Any]], Dict) -> Tuple[ref_resolver.Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[unicode, Any], ref_resolver.Loader]
+    # type: (Union[unicode, Dict[unicode, Any]], Dict) -> Tuple[ref_resolver.Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[unicode, Any], ref_resolver.Loader]
     metaschema_names, metaschema_doc, metaschema_loader = get_metaschema()
     if cache is not None:
         metaschema_loader.cache = cache
@@ -204,7 +204,7 @@ def load_and_validate(document_loader, avsc_names, document, strict):
 
 
 def validate_doc(schema_names, doc, loader, strict):
-    # type: (avro.schema.Names, Union[Dict[str, Any], List[Dict[str, Any]], str, unicode], ref_resolver.Loader, bool) -> None
+    # type: (avro.schema.Names, Union[Dict[unicode, Any], List[Dict[unicode, Any]], str, unicode], ref_resolver.Loader, bool) -> None
     has_root = False
     for r in schema_names.names.values():
         if ((hasattr(r, 'get_prop') and r.get_prop("documentRoot")) or (