check_isa_schemas updated to new json validator. #591

isatools/convert/isatab2cedar.py

@@ -8,10 +8,13 @@
 import os
 from os import listdir
 from os.path import isdir, join
+from pathlib import Path
 from uuid import uuid4
 
-from jsonschema import Draft4Validator, RefResolver
+from jsonschema import Draft4Validator, FormatChecker
 from jsonschema.exceptions import ValidationError
+from referencing import Registry
+from referencing.jsonschema import DRAFT4
 
 from isatools.io import isatab_parser
 
@@ -38,11 +41,24 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier):
         log.info("Converting ISA to CEDAR model for {}".format(work_dir))
         schema_file = "investigation_template.json"
         with open(join(CEDAR_SCHEMA_PATH, schema_file)) as json_fp:
-            schema = json.load(json_fp)
-        if schema is None:
+            investigation_schema = json.load(json_fp)
+        if investigation_schema is None:
             raise IOError("Could not load schema from {}".format(join(CEDAR_SCHEMA_PATH, schema_file)))
-        resolver = RefResolver("file://{}".format(join(CEDAR_SCHEMA_PATH, schema_file)), schema)
-        validator = Draft4Validator(schema, resolver=resolver)
+
+        resources = []
+        schemas_dir = Path(CEDAR_SCHEMA_PATH)
+        investigation_schema_path = Path(join(CEDAR_SCHEMA_PATH, schema_file))
+
+        for p in sorted(schemas_dir.glob("*.json")):
+            contents = json.loads(p.read_text(encoding="utf-8"))
+            resource = DRAFT4.create_resource(contents)
+            resources.append((p.resolve().as_uri(), resource))
+
+        registry = Registry().with_resources(resources)
+        main_uri = investigation_schema_path.resolve().as_uri()
+        print(registry.contents(main_uri))
+        schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"}
+        validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker())
 
         isa_tab = isatab_parser.parse(work_dir)
 
@@ -121,7 +137,7 @@ def createCEDARjson(self, work_dir, json_dir, inv_identifier):
                 study_identifier = ""
 
             try:
-                validator.validate(cedar_json, schema)
+                validator.validate(cedar_json)
             except ValidationError as e:
                 error_file_name = os.path.join(json_dir, "error.log")
                 with open(error_file_name, "w") as errorfile:

isatools/convert/isatab2json.py

@@ -8,9 +8,12 @@
 import re
 from enum import Enum
 from os.path import join
+from pathlib import Path
 from uuid import uuid4
 
-from jsonschema import Draft4Validator, RefResolver
+from jsonschema import Draft202012Validator, FormatChecker
+from referencing import Registry
+from referencing.jsonschema import DRAFT202012
 
 from isatools import isatab
 from isatools.io.isatab_parser import parse
@@ -139,10 +142,27 @@ def convert(self, work_dir):
 
             # validate json
             with open(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA)) as json_fp:
-                schema = json.load(json_fp)
-                resolver = RefResolver("file://" + join(SCHEMAS_PATH, INVESTIGATION_SCHEMA), schema)
-                validator = Draft4Validator(schema, resolver=resolver)
-                validator.validate(isa_json, schema)
+                # investigation_schema = json.load(json_fp)
+                # schema = DRAFT4.create_resource(investigation_schema)
+                # registry = Registry.with_resource(investigation_schema['id'], schema)
+                # validator = Draft4Validator(investigation_schema, registry=registry)
+                # validator.validate(isa_json)
+
+                resources = []
+                schemas_dir = Path(SCHEMAS_PATH)
+                investigation_schema_path = Path(join(SCHEMAS_PATH, INVESTIGATION_SCHEMA))
+
+                for p in sorted(schemas_dir.glob("*.json")):
+                    contents = json.loads(p.read_text(encoding="utf-8"))
+                    resource = DRAFT202012.create_resource(contents)
+                    resources.append((p.resolve().as_uri(), resource))
+
+                registry = Registry().with_resources(resources)
+                main_uri = investigation_schema_path.resolve().as_uri()
+                print(registry.contents(main_uri))
+                schema_ref = {"$ref": main_uri, "$schema": "https://json-schema.org/draft/2020-12/schema"}
+                validator = Draft202012Validator(schema_ref, registry=registry, format_checker=FormatChecker())
+                validator.validate(isa_json)
 
                 log.info("Conversion finished")
                 return isa_json

isatools/isajson/validate.py

@@ -13,8 +13,11 @@
 import os
 import re
 from io import StringIO
+from pathlib import Path
 
-from jsonschema import Draft4Validator, RefResolver, ValidationError
+from jsonschema import Draft202012Validator, FormatChecker, ValidationError
+from referencing import Registry
+from referencing.jsonschema import DRAFT202012
 
 from isatools.isajson.load import load
 
@@ -547,10 +550,21 @@ def check_isa_schemas(isa_json, investigation_schema_path):
     """Used for rule 0003 and 4003"""
     try:
         with open(investigation_schema_path) as fp:
-            investigation_schema = json.load(fp)
-            resolver = RefResolver("file://" + investigation_schema_path, investigation_schema)
-            validator = Draft4Validator(investigation_schema, resolver=resolver)
+            resources = []
+            investigation_schema_path = Path(investigation_schema_path)
+            schemas_dir = investigation_schema_path.parent
+
+            for p in sorted(schemas_dir.glob("*.json")):
+                contents = json.loads(p.read_text(encoding="utf-8"))
+                resource = DRAFT202012.create_resource(contents)
+                resources.append((p.resolve().as_uri(), resource))
+
+            registry = Registry().with_resources(resources)
+            main_uri = investigation_schema_path.resolve().as_uri()
+            schema_ref = {"$ref": main_uri, "$schema": "https://json-schema.org/draft/2020-12/schema"}
+            validator = Draft202012Validator(schema_ref, registry=registry, format_checker=FormatChecker())
             validator.validate(isa_json)
+
     except ValidationError as ve:
         errors.append({"message": "Invalid JSON against ISA-JSON schemas", "supplemental": str(ve), "code": 3})
         log.fatal("(F) The JSON does not validate against the provided ISA-JSON schemas!")

isatools/net/ols.py

@@ -39,6 +39,7 @@ def get_ols_ontologies():
                 file=file,
             )
         )
+
     return ontology_sources
 
 
@@ -47,6 +48,7 @@ def get_ols_ontology(ontology_name):
     ontologiesUri = OLS_API_BASE_URI + "/ontologies?size=" + str(OLS_PAGINATION_SIZE)
     log.debug(ontologiesUri)
     J = json.loads(urlopen(ontologiesUri).read().decode("utf-8"))
+    print("EMBEDDED: ", J["_embedded"]["ontologies"])
     ontology_sources = []
     for ontology_source_json in J["_embedded"]["ontologies"]:
         ontology_sources.append(
@@ -57,7 +59,10 @@ def get_ols_ontology(ontology_name):
                 file=ontology_source_json["_links"]["self"]["href"],
             )
         )
+    print("NAME: ", ontology_name)
+    print("SOURCES: ", [o.name for o in ontology_sources])
     hits = [o for o in ontology_sources if o.name == ontology_name]
+    print("HITS: ", hits)
     if len(hits) == 1:
         return hits[0]
     return None

isatools/net/storage_adapter.py

@@ -5,15 +5,18 @@
 import json
 import logging
 import os
-import pathlib
 from abc import ABCMeta, abstractmethod
 from io import BytesIO, StringIO
+from os.path import join
+from pathlib import Path
 from urllib.parse import urljoin
 from zipfile import ZipFile
 
 import requests
-from jsonschema import Draft4Validator, RefResolver
+from jsonschema import Draft4Validator, FormatChecker  # RefResolver
 from lxml import etree
+from referencing import Registry
+from referencing.jsonschema import DRAFT4
 
 log = logging.getLogger("isatools")
 
@@ -58,8 +61,22 @@ def validate_json_against_schema(json_dict, schema_src):
     """
     with open(schema_src) as schema_file:
         schema = json.load(schema_file)
-    resolver = RefResolver(pathlib.Path(os.path.abspath(schema_src)).as_uri(), schema)
-    validator = Draft4Validator(schema, resolver=resolver)
+        resources = []
+        schemas_dir = Path(schema_src)
+        investigation_schema_path = Path(join(schema_src, schema_file))
+
+        for p in sorted(schemas_dir.glob("*.json")):
+            contents = json.loads(p.read_text(encoding="utf-8"))
+            resource = DRAFT4.create_resource(contents)
+            resources.append((p.resolve().as_uri(), resource))
+
+        registry = Registry().with_resources(resources)
+        main_uri = investigation_schema_path.resolve().as_uri()
+        print(registry.contents(main_uri))
+        schema_ref = {"$ref": main_uri, "$schema": "http://json-schema.org/draft-04/schema"}
+        validator = Draft4Validator(schema_ref, registry=registry, format_checker=FormatChecker())
+
+    # validator = Draft4Validator(schema)
     return validator.validate(json_dict, schema)
 
 

isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json

@@ -1,5 +1,5 @@
 {
-  "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json",
+  "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/assay_schema.json",
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "title": "ISA Assay JSON Schema",
   "name": "ISA Assay JSON Schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/comment_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title": "ISA Comment schema - it corresponds to ISA Comment[] construct",
     "name" : "ISA Comment schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/data_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title": "ISA Data schema",
     "name" : "ISA Data schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title": "ISA Factor schema",
     "name": "ISA Factor schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/factor_value_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title": "ISA Factor Value schema",
     "name": "ISA Factor Value schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/investigation_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA investigation schema",
     "name" : "ISA Investigation schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA material attribute schema",
     "name" : "ISA Material Attribute schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_value_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_attribute_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA Material Attribute schema",
     "name" : "ISA Material Attribute schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json

@@ -1,5 +1,5 @@
 {
-  "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json",
+  "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/material_schema.json",
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "title" : "ISA Material schema",
   "name" : "ISA Material schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_annotation_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA ontology reference schema",
     "name" : "ISA ontology reference schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_reference_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/ontology_source_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA ontology source reference schema",
     "name" : "ISA ontology source reference schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/organization_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA Organization schema",
     "name" : "ISA Organization schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/person_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA Person schema",
     "name" :  "ISA Person schema",
@@ -12,7 +12,13 @@
         "lastName" : { "type" : "string"},
         "firstName" : { "type" : "string"},
         "midInitials" : { "type" : "string" },
-        "email" : { "type" : "string", "format" : "email"},
+        "email" : {
+          "anyOf": [
+            { "type" : "string", "format": "email" },
+            { "type" : "string", "maxLength": 0 },
+            { "type": "null" }
+          ]
+        },
         "phone" : { "type": "string"},
         "fax" : { "type" : "string" },
         "address" : { "type" : "string" },

isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_parameter_value_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA process parameter value schema",
     "name" : "ISA Process Parameter Value schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/process_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title": "ISA process or protocol application schema, corresponds to 'Protocol REF' columns in the study and assay files",
     "name" : "ISA Process schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_parameter_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA Protocol Parameter schema",
     "name" : "ISA Protocol Parameter schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/protocol_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title": "ISA Protocol schema",
     "name": "ISA Protocol schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/publication_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA Publication schema",
     "name" : "ISA Publication schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/sample_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA Sample schema",
     "name" : "ISA Sample schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json

@@ -1,5 +1,5 @@
 {
-    "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json",
+    "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/source_schema.json",
     "$schema": "https://json-schema.org/draft/2020-12/schema",
     "title" : "ISA Source schema",
     "name" : "ISA Source schema",

isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json

@@ -1,5 +1,5 @@
 {
-  "id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json",
+  "$id": "https://raw.githubusercontent.com/ISA-tools/isa-api/master/isatools/resources/schemas/isa_model_version_1_0_schemas/core/study_schema.json",
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "title": "ISA Study JSON schema",
   "name" : "ISA Study JSON schema",

isatools/utils.py

@@ -14,7 +14,6 @@
 from zipfile import ZipFile
 
 import pandas as pd
-import yaml
 
 from isatools import isatab
 from isatools.model import Process