Update UDS Elasticsearch schema to include granule metadata

[ngachung]

Open Question: Do we want to duplicate Cumulus granule metadata into UDS ES or query both Cumulus and UDS ES?

Two new indices

• Tenant custom fields
• Granule metadata

Currently we have two indices

• Collection
  Collection ID, bounding box, start and end time
  [TODO] Need a job to update collection start and end time based on granules metadata

• Authorization mapping
  Tenant, Tenant Venue, Action, Collection Map, User Group

[wphyojpl]

Example incoming schema:

{
  "activity_id_rtt": {
    "type": "keyword"
  },
  "activity_name_rtt": {
    "type": "keyword",
    "normalizer": "ocs_std_normalizer"
  },
  "activity_notes_rtt": {
    "type": "text"
  },
  "target_id_rtt": {
    "type": "keyword"
  },
  "target_name_rtt":  {
    "type": "keyword",
    "normalizer": "ocs_std_normalizer"
  },
  "ext": {
    "type": "keyword"
  },
  "spec_flag": {
    "type": "keyword"
  },
  "rct": {
    "type": "date"
  },
  "product_version": {
    "type": "short"
  },
  "seq_id_rtt": {
    "type": "keyword"
  },
  "year": {
    "type": "integer"
  },
  "ingester_lambda_version": {
    "type": "keyword"
  },
  "instrument_id": {
    "type": "keyword"
  },
  "time1_doy": {
    "type": "boolean"
  },
  "time1": {
    "type": "long"
  },
  "mission_venue": {
    "type": "keyword"
  },
  "time2": {
    "type": "long"
  },
  "site": {
    "type": "integer"
  },
  "marsviewer_version": {
    "type": "keyword"
  },
  "provider": {
    "type": "keyword"
  },
  "product_id": {
    "type": "keyword"
  },
  "drive": {
    "type": "integer"
  },
  "config": {
    "type": "keyword"
  },
  "pds_label": {
    "dynamic": "false",
    "properties": {
      "BOOT_COUNT": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "ACTIVE_FLIGHT_STRING_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "APPLICATION_PROCESS_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "APPLICATION_PROCESS_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "AUTO_DELETE_FLAG": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "COMMAND_SEQUENCE_NUMBER": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "COMMUNICATION_SESSION_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "DATA_SET_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "DATA_SET_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "DOWNLOAD_PRIORITY": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "EARTH_RECEIVED_START_TIME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "EXPECTED_PACKETS": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "EXPECTED_TRANSMISSION_PATH": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "FLIGHT_SOFTWARE_MODE": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "FLIGHT_SOFTWARE_VERSION_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "INSTRUMENT_HOST_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "INSTRUMENT_HOST_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "INSTRUMENT_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "INSTRUMENT_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "INSTRUMENT_TYPE": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "LOCAL_MEAN_SOLAR_TIME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "LOCAL_TRUE_SOLAR_TIME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "LOCAL_TRUE_SOLAR_TIME_SOL": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "MISSION_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "MISSION_PHASE_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "OBSERVATION_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "PLANET_DAY_NUMBER": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "PRODUCT_COMPLETION_STATUS": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "PRODUCT_CREATION_TIME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "PRODUCT_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "PRODUCT_TAG": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "PRODUCT_TYPE": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "PRODUCT_VERSION_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "RECEIVED_PACKETS": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "RELEASE_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "REQUEST_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "ROVER_MOTION_COUNTER": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "ROVER_MOTION_COUNTER_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "SEQUENCE_EXECUTION_COUNT": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "SEQUENCE_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "SEQUENCE_VERSION_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "SOLAR_LONGITUDE": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "float"
      },
      "SPACECRAFT_CLOCK_CNT_PARTITION": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "SPACECRAFT_CLOCK_START_COUNT": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "float"
      },
      "SPACECRAFT_CLOCK_STOP_COUNT": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "float"
      },
      "SPICE_FILE_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "START_TIME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "STOP_TIME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "TARGET_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "TARGET_TYPE": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "TELEMETRY_PROVIDER_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "TELEMETRY_SOURCE_CHECKSUM": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "TELEMETRY_SOURCE_HOST_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "TELEMETRY_SOURCE_NAME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "TELEMETRY_SOURCE_SCLK_START": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "float"
      },
      "TELEMETRY_SOURCE_SIZE": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "TELEMETRY_SOURCE_START_TIME": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "format": "yyyy-MM-dd'T'HH:mm:ss.SSSZ||strict_date_optional_time||epoch_millis",
        "ignore_malformed": true,
        "type": "date"
      },
      "TELEMETRY_SOURCE_TYPE": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "type": "text"
      },
      "TRANSMISSION_PATH": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      },
      "VIRTUAL_CHANNEL_ID": {
        "fields": {
          "keyword": {
            "type": "keyword"
          }
        },
        "ignore_malformed": true,
        "type": "integer"
      }
    }
  }
}

[ngachung]

Done as part of #201