Update Data Script for US Core v4/v5

.gitignore

@@ -1,3 +1,4 @@
 # Mac OS X FS Junk
 **/.DS_Store
-output/
+output/
+lib/validator_cli.jar

README.md

@@ -12,6 +12,7 @@ This uscore-data-script requires working installations of Ruby and Java. After c
 ```
 cd <path>/uscore-data-script
 bundle install
+get_validator.sh
 ```
 
 ## Creating the Data Set
@@ -22,14 +23,17 @@ The uscore-data-script will execute Synthea to generate synthetic patients, load
 FHIR Bundles, and select certain patients based on criteria described below.
 
 ```
-bundle exec ruby uscore-data-script.rb [mrburns]
+bundle exec ruby uscore-data-script.rb [mrburns] [version]
 ```
 
 There is an optional `mrburns` parameter, which if included, will generate a single longitudinal
 patient who possesses at least one resource conforming to every US Core profile. In other words,
 one patient with everything. If the `mrburns` parameter is not included, the script will generate
 a small collection of testing patients.
 
+There is an optional `version` parameter. Legal values are `v3`, `v4`, and `v5` to specify a version of
+the US Core Implementation Guide. The default value is `v4`.
+
 2. Use Data
 
 Use the data files located in `/output/data`.  It includes 
@@ -73,7 +77,7 @@ on Pediatric BMI and Pediatric Weight, because the `Observation.value[x]` are re
 
 # License
 
-Copyright 2020 The MITRE Corporation
+Copyright 2020 - 2022 The MITRE Corporation
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

get_validator.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+
+wget https://github.com/hapifhir/org.hl7.fhir.core/releases/download/5.6.54/validator_cli.jar --no-check-certificate
+mv validator_cli.jar lib/validator_cli.jar

lib/choice_type_creator.rb

@@ -10,52 +10,8 @@ module DataScript
   # automatically generate.
   class ChoiceTypeCreator
     SWAP_COUNT = 1
-    MUST_SUPPORT_CHOICE_TYPES = {
-      FHIR::DiagnosticReport =>
-      {
-        prefix: 'effective',
-        suffixes: %w[DateTime Period],
-        profiles: [
-          'http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-note',
-          'http://hl7.org/fhir/us/core/StructureDefinition/us-core-diagnosticreport-lab'
-        ]
-      },
-      FHIR::Immunization =>
-      {
-        prefix: 'occurrence',
-        suffixes: %w[DateTime String],
-        profiles: [
-          'http://hl7.org/fhir/us/core/StructureDefinition/us-core-immunization'
-        ]
-      },
-      FHIR::Observation =>
-      {
-        prefix: 'effective',
-        suffixes: %w[DateTime Period],
-        profiles: [
-          'http://hl7.org/fhir/StructureDefinition/resprate',
-          'http://hl7.org/fhir/StructureDefinition/heartrate',
-          'http://hl7.org/fhir/StructureDefinition/bodyweight',
-          'http://hl7.org/fhir/StructureDefinition/bp',
-          'http://hl7.org/fhir/us/core/StructureDefinition/us-core-smokingstatus',
-          'http://hl7.org/fhir/us/core/StructureDefinition/pediatric-weight-for-height',
-          'http://hl7.org/fhir/us/core/StructureDefinition/us-core-observation-lab',
-          'http://hl7.org/fhir/us/core/StructureDefinition/pediatric-bmi-for-age',
-          'http://hl7.org/fhir/us/core/StructureDefinition/us-core-pulse-oximetry',
-          'http://hl7.org/fhir/us/core/StructureDefinition/head-occipital-frontal-circumference-percentile',
-          'http://hl7.org/fhir/StructureDefinition/bodyheight',
-          'http://hl7.org/fhir/StructureDefinition/bodytemp'
-        ]
-      },
-      FHIR::Procedure =>
-      {
-        prefix: 'performed',
-        suffixes: %w[DateTime Period],
-        profiles: [
-          'http://hl7.org/fhir/us/core/StructureDefinition/us-core-procedure'
-        ]
-      }
-    }.freeze
+    # MUST_SUPPORT_CHOICE_TYPES are defined in ./v3/choice_types.rb, ./v4/choice_types.rb, or ./v5/choice_types.rb
+
 
     # Goes through the set of bundles, and ensure we have at least SWAP_COUNT resources
     # in each resource/profile pair listed above that conform to each choice type.
@@ -90,15 +46,29 @@ def self.check_choice_types(bundles)
           # If we still have choice types left over, we need to add some of this type
           next if missing_attrs.empty?
 
+          # puts "Profile: #{profile_url}"
           missing_attrs.each do |missing_attr|
+            # puts " - Missing Attr: #{missing_attr}"
             # If there are more than (SWAP_COUNT * the number of choices) resources of this class
             # Swap the choice type on SWAP_COUNT to the new type
             profile_resources.select { |resource| missing_attrs.none? { |attribute| resource.send(attribute) } }.first(SWAP_COUNT).each do |resource|
               # Get the first non-nil attribute value
-              old_attr_type, old_attr_val = all_attrs.map do |a|
-                av = resource.send(a)
-                [a, av] if av
-              end.compact.first
+              old_attr_type = all_attrs.find do |type|
+                # puts "    > Try #{type}"
+                !resource.send(type).nil?
+              end
+              # puts "    >> Found #{old_attr_type}"
+              old_attr_val = nil
+              old_attr_val = resource.send(old_attr_type) unless old_attr_type.nil?
+              # puts "    >>> Value #{old_attr_val}"
+              if old_attr_type.nil? || old_attr_val.nil?
+                # puts "    >>>> This type is nil, try #{choice[:prefix]}"
+                old_attr_val = resource.send(choice[:prefix])
+                # puts "    >>>> Value #{old_attr_val}"
+                old_attr_type = old_attr_val.class.to_s.split('::').last
+                old_attr_type = "#{choice[:prefix]}#{old_attr_type}"
+                # puts "    >>>> Assumed Type #{old_attr_type}"
+              end
               # convert it (using some ugly if statements) to the new type
               new_val = convert_choice_types(old_attr_val, missing_attr, old_attr_type)
               if profile_resources.count > (SWAP_COUNT * all_attrs.count)
@@ -190,7 +160,7 @@ def self.datetime_plus_1_hour(datetime_string)
     # @param bundles [Array] an array of FHIR::Bundle objects
     # @return nil
     def self.add_to_correct_bundle(resource, bundles)
-      bundle = bundles.find { |b| DataScript::Constraints.patient(b).id == get_resource_patient_id(resource) }
+      bundle = bundles.find { |b| DataScript::Constraints.patient(b)&.id == get_resource_patient_id(resource) }
       entry = create_bundle_entry(resource)
       bundle.entry << entry
       provenance = bundle.entry.find { |e| e.resource.is_a? FHIR::Provenance }.resource

lib/filter.rb

@@ -0,0 +1,121 @@
+require 'pry'
+
+module DataScript
+  class Filter
+    def self.filter!(bundles)
+      organization_bundle = bundles.find {|b| b.entry.first.resource.resourceType == 'Organization'}
+      practitioner_bundle = bundles.find {|b| b.entry.first.resource.resourceType == 'Practitioner'}
+      patient_bundles = bundles.select {|b| b.entry.first.resource.resourceType == 'Patient'}
+
+      patient_bundles.each do |bundle|
+        print "\n"
+        tik = Time.now.to_i
+        initial_length = bundle.entry.length
+        profile_coverage = []
+        encounters_to_keep = []
+        resources_to_keep = []
+
+        bundle.entry.reverse.each do |entry|
+          next unless ['AllergyIntolerance','Device','Encounter','Goal','RelatedPerson'].include?(entry.resource.resourceType)
+          # start by looking at Encounters, most but not all resources reference the Encounter...
+          if entry.resource.resourceType == 'Encounter'
+            print '.'
+            encounter_resources = get_resources_associated_with_encounter(bundle, entry.fullUrl)
+            encounter_resources_profiles = encounter_resources.map {|resource| resource&.meta&.profile&.first}.compact
+            if encounter_resources_profiles.any? {|p| !profile_coverage.include?(p) }
+              profile_coverage.append(encounter_resources_profiles).flatten!
+              profile_coverage.uniq!
+              encounters_to_keep << entry.resource
+              resources_to_keep.append(encounter_resources).flatten!
+            end
+          elsif entry.resource.resourceType == 'AllergyIntolerance' # allergyintolerance is special because it does not reference an encounter
+            print 'a'
+            resource_profile = entry.resource&.meta&.profile&.first
+            if resource_profile == 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-allergyintolerance'
+              resources_to_keep << entry.resource
+              profile_coverage.append(resource_profile).flatten!
+              profile_coverage.uniq!
+            end
+          elsif entry.resource.resourceType == 'Device' # device is special because it does not reference an encounter
+            print 'd'
+            resource_profile = entry.resource&.meta&.profile&.first
+            if resource_profile == 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-implantable-device'
+              resources_to_keep << entry.resource
+              profile_coverage.append(resource_profile).flatten!
+              profile_coverage.uniq!
+            end
+          elsif entry.resource.resourceType == 'Goal' # goal is special because it does not reference an encounter
+            print 'g'
+            resource_profile = entry.resource&.meta&.profile&.first
+            if resource_profile == 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-goal'
+              resources_to_keep << entry.resource
+              profile_coverage.append(resource_profile).flatten!
+              profile_coverage.uniq!
+            end
+          elsif entry.resource.resourceType == 'RelatedPerson' # relatedperson is special because it does not reference anything
+            print 'r'
+            resource_profile = entry.resource&.meta&.profile&.first
+            if resource_profile == 'http://hl7.org/fhir/us/core/StructureDefinition/us-core-relatedperson'
+              resources_to_keep << entry.resource
+              profile_coverage.append(resource_profile).flatten!
+              profile_coverage.uniq!
+            end
+          end
+        end
+        print "\n"
+
+        bundle.entry.keep_if do |entry|
+          ['Patient','Provenance'].include?(entry.resource.resourceType) ||
+          encounters_to_keep.include?(entry.resource) ||
+          resources_to_keep.include?(entry.resource)
+        end
+
+        provenance = bundle.entry.find {|e| e.resource.resourceType == 'Provenance' }.resource
+        uuids = bundle.entry.map {|e| e.fullUrl}
+        provenance.target.keep_if {|reference| uuids.include?(reference.reference) }
+        final_length = bundle.entry.length
+        tok = Time.now.to_i
+        puts "  - Filtered #{initial_length} resources down to #{final_length} (#{DataScript::TimeUtilities.pretty(tok - tik)})."
+      end
+    end
+
+    def self.get_resources_associated_with_encounter(bundle, encounter_urn)
+      resources = []
+      refd_medications = []
+      refd_conditions = []
+      refd_related_person = []
+
+      bundle.entry.each do |entry|
+        if entry.resource.respond_to?(:encounter) && entry.resource.encounter&.reference == encounter_urn
+          resources << entry.resource
+          refd_medications << entry.resource&.medicationReference&.reference if entry.resource.respond_to?(:medicationReference)
+          refd_conditions << entry.resource&.reasonReference&.first&.reference if entry.resource.respond_to?(:reasonReference)
+          refd_related_person << entry.resource&.participant&.find {|x| x&.role&.first&.text = 'Caregiver (person)'}&.member&.reference if entry.resource.resourceType == 'CareTeam'
+        elsif entry.resource.resourceType == 'DocumentReference' && entry.resource.context.encounter.first.reference == encounter_urn
+          resources << entry.resource
+        end
+      end
+
+      refd_medications.uniq!
+      refd_conditions.uniq!
+      refd_related_person.uniq!
+
+      refd_medications.each do |fullUrl|
+        resources << bundle.entry.find {|e| e.fullUrl == fullUrl}&.resource
+      end
+
+      refd_conditions.each do |fullUrl|
+        resources << bundle.entry.find {|e| e.fullUrl == fullUrl}&.resource
+      end
+
+      refd_related_person.each do |fullUrl|
+        resources << bundle.entry.find {|e| e.fullUrl == fullUrl}&.resource
+      end
+
+      resources.uniq!
+      resources.compact!
+      resources
+    end
+
+  end
+end