Fix edge case and improve unit tests (#100)

.gitignore

@@ -3,3 +3,5 @@ main_test.py
 sample_test_data/
 # Distribution / packaging
 *.egg-info/
+
+tests/TestData

datasimulator/generator.py

@@ -2,8 +2,13 @@
 import rstr
 import random
 
+from cdislogging import get_logger
+
 from .errors import UserError
 
+
+logger = get_logger("data-simulator generator", log_level="info")
+
 tried_words = False
 WORDS = None
 
@@ -18,8 +23,11 @@ def generate_string_data(size=10, pattern=None, format=None):
         try:
             word_file = "/usr/share/dict/words"
             WORDS = open(word_file).read().splitlines()
-        except Exception:
-            pass
+            logger.info("Using '/usr/share/dict/words' for string generation.")
+        except Exception as e:
+            logger.info(
+                f"Unable to use '/usr/share/dict/words' for string generation. Generating random strings instead. Details: {e}"
+            )
     if pattern or not WORDS:
         pattern = pattern or "^[0-9a-f]{" + str(size) + "}"
         return rstr.xeger(pattern)

datasimulator/graph.py

@@ -217,14 +217,25 @@ def generate_submission_order_path_to_node(self, node, cmc_node=None):
             submission_order.append(cmc_node)
         index = 0
 
+        project_node = None
         while index < len(submission_order):
             cur_node = submission_order[index]
             index += 1
             if not cur_node:
                 continue
             for linked_node_dict in cur_node.required_links:
-                if linked_node_dict["node"] not in submission_order:
-                    submission_order.append(linked_node_dict["node"])
+                if linked_node_dict["node"].name == "project":
+                    project_node = linked_node_dict["node"]
+                    continue
+                if linked_node_dict["node"] in submission_order:
+                    # reorder to place the node at the beginning. eg. if this node is parent to 2 other
+                    # nodes, we need it to be submitted before the 2 others, and not in-between.
+                    submission_order.remove(linked_node_dict["node"])
+                submission_order.append(linked_node_dict["node"])
+
+        if project_node:  # project should always be submitted first
+            submission_order.append(project_node)
+
         submission_order.reverse()
 
         return submission_order

tests/conftest.py

@@ -7,6 +7,6 @@
 
 
 @pytest.fixture
-def init_dictionary():
-    datadictionary = DataDictionary(root_dir=os.path.join(MOD_DIR, "schemas"))
+def default_dictionary():
+    datadictionary = DataDictionary(root_dir=os.path.join(MOD_DIR, "schemas/default"))
     dictionary.init(datadictionary)

tests/schemas/_definitions.yaml → tests/schemas/default/_definitions.yaml

@@ -58,7 +58,7 @@ file_name:
 
 file_size:
     type: integer
-    term: 
+    term:
         $ref: "_terms.yaml#/file_size"
 
 file_format:
@@ -105,7 +105,7 @@ data_file_error_type:
 
 state:
     term:
-        $ref: "_terms.yaml#/state" 
+        $ref: "_terms.yaml#/state"
     default: validated
     downloadable:
         - uploaded
@@ -136,7 +136,7 @@ state:
 
 file_state:
     term:
-        $ref: "_terms.yaml#/file_state" 
+        $ref: "_terms.yaml#/file_state"
     default: registered
     enum:
         - registered
@@ -177,7 +177,7 @@ data_file_properties:
   file_size:
     $ref: "#/file_size"
   md5sum:
-    $ref: "#/md5sum" 
+    $ref: "#/md5sum"
   file_state:
     $ref: "#/file_state"
   object_id:
@@ -188,7 +188,7 @@ data_file_properties:
     $ref: "#/data_file_error_type"
   state_comment:
     type: string
-    description: > 
+    description: >
       Optional comment about why the file is in the
       current state, mainly for invalid state.
   project_id:

tests/schemas/_terms.yaml → tests/schemas/default/_terms.yaml

@@ -288,7 +288,7 @@ biomarker_name:
     term: Biomarker Name
     source: caDSR
     cde_id: 5473
-    cde_version: 11.0 
+    cde_version: 11.0
     term_url: "https://cdebrowser.nci.nih.gov/CDEBrowser/search?elementDetails=9&FirstTimer=0&PageId=ElementDetailsGroup&publicId=5473&version=2.31"
 
 biomarker_result:
@@ -618,7 +618,7 @@ dlco_ref_predictive_percent:
     cde_version: 1.0
     term_url: "https://cdebrowser.nci.nih.gov/CDEBrowser/search?elementDetails=9&FirstTimer=0&PageId=ElementDetailsGroup&publicId=2180255&version=1.0"
 
-encoding: 
+encoding:
   description: >
     Version of ASCII encoding of quality values found in the file.
   termDef:

tests/schemas/acknowledgement.yaml → tests/schemas/default/acknowledgement.yaml

@@ -1,7 +1,7 @@
 $schema: "http://json-schema.org/draft-04/schema#"
 
 id: "acknowledgement"
-title: Acknowledgement 
+title: Acknowledgement
 type: object
 namespace: http://gdc.nci.nih.gov
 category: administrative
@@ -21,7 +21,7 @@ systemProperties:
 
 links:
   - name: projects
-    backref: acknowledgements 
+    backref: acknowledgements
     label: contribute_to
     target_type: project
     multiplicity: many_to_many

tests/schemas/case.yaml → tests/schemas/default/case.yaml

@@ -9,7 +9,7 @@ program: '*'
 project: '*'
 description: >
   The collection of all data related to a specific subject in the
-  context of a specific experiment. 
+  context of a specific experiment.
 additionalProperties: false
 submittable: true
 validators: null
@@ -22,7 +22,7 @@ systemProperties:
   - state
 
 links:
-  - name: experiments 
+  - name: experiments
     backref: cases
     label: member_of
     target_type: experiment
@@ -31,7 +31,7 @@ links:
 
 required:
   - submitter_id
-  - experiments 
+  - experiments
 
 uniqueKeys:
   - [id]
@@ -56,7 +56,7 @@ properties:
   disease_type:
     description: "Name of the disease for the case."
     type: string
-  experiments: 
+  experiments:
     $ref: "_definitions.yaml#/to_one"
   project_id:
     $ref: "_definitions.yaml#/project_id"

tests/schemas/clinical_test.yaml → tests/schemas/default/clinical_test.yaml

@@ -1,16 +1,16 @@
 $schema: "http://json-schema.org/draft-04/schema#"
 
 id: "clinical_test"
-title: Clinical Test 
+title: Clinical Test
 type: object
 namespace: http://gdc.nci.nih.gov
-category: clinical 
+category: clinical
 project: '*'
 program: '*'
-description: > 
-  Metadata concerning any clinical tests used in relation to a case diagnosis. 
+description: >
+  Metadata concerning any clinical tests used in relation to a case diagnosis.
 additionalProperties: false
-submittable: true 
+submittable: true
 validators: null
 
 systemProperties:
@@ -21,9 +21,9 @@ systemProperties:
   - state
 
 links:
-  - name: cases 
+  - name: cases
     backref: clinical_tests
-    label: performed_for 
+    label: performed_for
     target_type: case
     multiplicity: many_to_one
     required: true
@@ -172,7 +172,7 @@ properties:
       $ref: "_terms.yaml#/her2_erbb2_result_fish"
     enum:
       - Negative
-      - Not Performed 
+      - Not Performed
       - Positive
       - Unknown
 
@@ -233,7 +233,7 @@ properties:
   diagnoses:
     $ref: "_definitions.yaml#/to_many"
   project_id:
-    $ref: "_definitions.yaml#/project_id" 
+    $ref: "_definitions.yaml#/project_id"
   created_datetime:
     $ref: "_definitions.yaml#/datetime"
   updated_datetime:

tests/schemas/core_metadata_collection.yaml → tests/schemas/default/core_metadata_collection.yaml

@@ -65,7 +65,7 @@ properties:
 
   format:
     description: >
-      The file format, physical medium, or dimensions of the resource. Examples of dimensions include size and duration. Recommended best practice is to use a controlled vocabulary such as the list of Internet Media Types [MIME] (http://www.iana.org/assignments/media-types/). 
+      The file format, physical medium, or dimensions of the resource. Examples of dimensions include size and duration. Recommended best practice is to use a controlled vocabulary such as the list of Internet Media Types [MIME] (http://www.iana.org/assignments/media-types/).
     type: string
 
   language:
@@ -110,4 +110,3 @@ properties:
 
   projects:
     $ref: "_definitions.yaml#/to_one"
-

tests/schemas/diagnosis.yaml → tests/schemas/default/diagnosis.yaml

@@ -382,8 +382,8 @@ properties:
     term:
       $ref: "_terms.yaml#/cause_of_death"
     enum:
-      - Cancer Related 
-      - Not Cancer Related 
+      - Cancer Related
+      - Not Cancer Related
       - Unknown
 
   circumferential_resection_margin:
@@ -500,7 +500,7 @@ properties:
       $ref: "_terms.yaml#/hiv_positive"
     enum:
       - "Yes"
-      - "No" 
+      - "No"
       - Unknown
 
   hpv_positive_type:
@@ -559,7 +559,7 @@ properties:
   lymph_nodes_positive:
     term:
       $ref: "_terms.yaml#/lymph_nodes_positive"
-    type: integer 
+    type: integer
 
   lymphatic_invasion_present:
     term:

tests/schemas/experiment.yaml → tests/schemas/default/experiment.yaml

@@ -25,7 +25,7 @@ links:
   - name: projects
     backref: experiments
     label: performed_for
-    target_type: project 
+    target_type: project
     multiplicity: many_to_one
     required: true
 

tests/schemas/project.yaml → tests/schemas/default/project.yaml

@@ -10,7 +10,7 @@ description: >
   Any specifically defined piece of work that is undertaken or attempted to meet a single
   requirement. (NCIt C47885)
 additionalProperties: false
-submittable: true 
+submittable: true
 validators: null
 
 systemProperties:
@@ -21,8 +21,8 @@ systemProperties:
   - intended_release_date
 
 required:
-  - code 
-  - name 
+  - code
+  - name
   - programs
 
 uniqueKeys:

tests/schemas/publication.yaml → tests/schemas/default/publication.yaml

@@ -33,11 +33,11 @@ required:
 
 uniqueKeys:
   - [ id ]
-  - [ project_id, submitter_id ] 
+  - [ project_id, submitter_id ]
 
 properties:
   type:
-    enum: [ "publication" ] 
+    enum: [ "publication" ]
   id:
     $ref: "_definitions.yaml#/UUID"
     systemAlias: node_id

tests/schemas/slide_count.yaml → tests/schemas/default/slide_count.yaml

@@ -70,7 +70,7 @@ properties:
       - Cytoplasmic
       - Both
       - None
-      - Not Determined 
+      - Not Determined
   frame_identifier:
     description: "Name, number, or other identifier given to the frame of the slide from which this image was taken."
     type: string

tests/schemas/slide_image.yaml → tests/schemas/default/slide_image.yaml

@@ -73,7 +73,7 @@ properties:
   data_format:
     term:
       $ref: "_terms.yaml#/data_format"
-    type: string 
+    type: string
   experimental_strategy:
     description: "Classification of the slide type with respect to its experimental use."
     enum:

tests/schemas/submitted_copy_number.yaml → tests/schemas/default/submitted_copy_number.yaml

@@ -64,15 +64,15 @@ properties:
   data_type:
     term:
       $ref: "_terms.yaml#/data_type"
-    type: string 
+    type: string
   data_format:
     term:
       $ref: "_terms.yaml#/data_format"
-    type: string 
+    type: string
   experimental_strategy:
     term:
       $ref: "_terms.yaml#/experimental_strategy"
-    type: string 
+    type: string
   aliquots:
     $ref: "_definitions.yaml#/to_one"
   read_groups: