metadata handling

R/get_eurostat_json.R

@@ -183,7 +183,6 @@ get_eurostat_json <- function(id,
   result <- httr2::resp_body_json(
       resp = resp,
       simplifyVector = TRUE)
-
   if (httr2::resp_is_error(resp)) {
 
     # These objects are only needed if there is an error
@@ -203,10 +202,10 @@ get_eurostat_json <- function(id,
                              "  Error id: {id} ({faultstring})\n",
                              "  Error label from API: {label}")
       )
-    } else {
+    } else{
       stop(stringr::str_glue("\n",
                              "HTTP status: {status} ({status_code_label})\n",
-                             "  Error id: {id} ({faultstring})\n",
+                             "  Error id: {id} \n",
                              "  Error label from API: {label}")
       )
     }

R/get_eurostat_sdmx.R

@@ -122,6 +122,98 @@ get_eurostat_sdmx <- function(
 
 }
 
+extract_metadata <- function(id, agency = "Eurostat") {
+
+  api_base_uri <- build_api_base_uri(agency)
+
+  data_structure_definition_url <- paste0(
+    api_base_uri,
+    "/sdmx/2.1/dataflow/estat/",
+    id)
+
+  dsd_xml <- xml2::read_xml(data_structure_definition_url)
+
+  # Define namespaces
+  namespaces <- xml2::xml_ns(dsd_xml)
+
+  # Extract Header information
+  header <- xml2::xml_find_first(dsd_xml, ".//m:Header", namespaces)
+  header_id <- xml2::xml_text(xml2::xml_find_first(header, ".//m:ID", namespaces))
+  prepared <- substr(xml2::xml_text(xml2::xml_find_first(header, ".//m:Prepared", namespaces)),1,10)
+  sender_id <- xml2::xml_attr(xml2::xml_find_first(header, ".//m:Sender", namespaces), "id")
+
+  # Continue with dataflow and annotations extraction 
+  dataflow <- xml2::xml_find_first(dsd_xml, ".//s:Dataflow", namespaces)
+  dataflow_id <- xml2::xml_attr(dataflow, "id")
+  urn <- xml2::xml_attr(dataflow, "urn")
+  agencyID <- xml2::xml_attr(dataflow, "agencyID")
+  version <- xml2::xml_attr(dataflow, "version")
+  isFinal <- xml2::xml_attr(dataflow, "isFinal")
+  # Extract names in different languages
+  # Extract names in different languages independently
+  name_de <- xml2::xml_text(xml2::xml_find_first(dataflow, ".//c:Name[@xml:lang='de']", namespaces))
+  name_en <- xml2::xml_text(xml2::xml_find_first(dataflow, ".//c:Name[@xml:lang='en']", namespaces))
+  name_fr <- xml2::xml_text(xml2::xml_find_first(dataflow, ".//c:Name[@xml:lang='fr']", namespaces))
+
+  source_institutions <- list()
+  doi_details <- NULL
+
+  annotations_nodes <- xml2::xml_find_all(dataflow, ".//c:Annotation", namespaces)
+  for (node in annotations_nodes) {
+    title <- xml2::xml_text(xml2::xml_find_first(node, ".//c:AnnotationTitle", namespaces))
+    type <- xml2::xml_text(xml2::xml_find_first(node, ".//c:AnnotationType", namespaces))
+    texts_nodes <- xml2::xml_find_all(node, ".//c:AnnotationText", namespaces)
+
+
+    # Assign specific annotations based on type
+    if (type == "OBS_PERIOD_OVERALL_LATEST") {
+      latest_period_timestamp <- title  # Directly store the latest period timestamp
+    } else if (type == "OBS_PERIOD_OVERALL_OLDEST") {
+      oldest_period_timestamp <- title
+    } else if (type == "UPDATE_DATA") {
+      update_data_timestamp <- title
+    } else if(type == "SOURCE_INSTITUTIONS"){
+      for (text_node in texts_nodes) {
+        lang <- xml2::xml_attr(text_node, "xml:lang")
+        text <- xml2::xml_text(text_node)
+        source_institutions[[lang]] <- text
+
+      }
+
+    }
+  }
+
+  # Extract DOI URL if the annotation contains adms:Identifier
+  if (grepl("adms:Identifier", title)) {
+    title_xml <- xml2::read_xml(title)
+    doi_url <- xml2::xml_attr(xml2::xml_find_first(title_xml, ".//adms:Identifier"), "rdf:about", xml2::xml_ns(title_xml))
+  }
+
+
+  metadata <- list(
+    Name_EN = name_en,
+    Name_DE = name_de,
+    Name_FR = name_fr,
+    DOI_URL = doi_url,
+    DataflowID = dataflow_id,
+    AgencyID = agencyID,
+    ID = header_id,
+    Prepared = prepared,
+    SenderID = sender_id,
+    OldestPeriodTimestamp = oldest_period_timestamp,
+    LatestPeriodTimestamp = latest_period_timestamp,
+    UpdateDataTimestamp = update_data_timestamp,
+    SourceInstitutions = source_institutions,
+    URN = urn,
+    Version = version,
+    IsFinal = isFinal
+  )
+
+  return(metadata)
+
+}
+
+
 legacy_data_format <- function(x, cols_to_drop = c("DATAFLOW", "LAST.UPDATE", "freq")) {
   x <- x[setdiff(names(x), cols_to_drop)]
   cols_to_rename <- data.frame(old = c("TIME_PERIOD", "OBS_VALUE"),