OECD import scripts (datacommonsorg#867)

* oecd scripts

* fix

* update readme

Author: n-h-diaz

scripts/oecd/sdmx/README.md

@@ -0,0 +1,30 @@
+# OECD Bulk Import
+
+This folder contains scripts for the bulk OECD import. This is currently a schemaless import.
+
+Note: This was a very quick first pass attempt to get some data in, so only contains a few hundred of the OECD datasets that follow a specific format.
+
+TODO(nhdiaz): Add tests / get remaining data.
+
+To download data: 
+```
+OPENSSL_CONF=openssl.cnf python3 download.py
+```
+
+To process data and generate artifacts:
+```
+python3 process.py
+```
+
+## SDMX
+
+OECD uses the SDMX format for their data. We have translated this into the Data Commons data model as follows:
+
+* The LOCATION dimension is used for observationAbout
+* The TIME_PERIOD dimension is used for observationDate
+* The TIME_FORMAT attribute is used for observationPeriod
+* The UNIT attribute is used for unit
+* The POWERCODE attribute is used for scalingFactor
+* The OBS_STATUS (observation level) attribute is used for measurementMethod
+* All other dimensions and attributes are added the stat var definition
+* The series is used as the parent SVG

scripts/oecd/sdmx/countries

@@ -0,0 +1,265 @@
+ABW
+AFG
+AFI
+AGO
+AIA
+ALB
+AND
+ANT
+ARE
+ARG
+ARM
+ASM
+ATA
+ATB
+ATF
+ATG
+ATN
+AUS
+AUT
+AZE
+BDI
+BEL
+BEN
+BES
+BFA
+BGD
+BGR
+BHR
+BHS
+BIH
+BLM
+BLR
+BLZ
+BMU
+BOL
+BRA
+BRB
+BRN
+BTN
+BVT
+BWA
+BYS
+CAF
+CAN
+CCK
+CHE
+CHL
+CHN
+CIV
+CMR
+COD
+COG
+COK
+COL
+COM
+CPV
+CRI
+CSK
+CTE
+CUB
+CUW
+CXR
+CYM
+CYP
+CZE
+DDR
+DEU
+DHY
+DJI
+DMA
+DNK
+DOM
+DZA
+ECU
+EGY
+ERI
+ESH
+ESP
+EST
+ETH
+FIN
+FJI
+FLK
+FRA
+FRO
+FSM
+FXX
+GAB
+GBR
+GEL
+GEO
+GGY
+GHA
+GIB
+GIN
+GLP
+GMB
+GNB
+GNQ
+GRC
+GRD
+GRL
+GTM
+GUF
+GUM
+GUY
+HKG
+HMD
+HND
+HRV
+HTI
+HUN
+HVO
+IDN
+IMN
+IND
+IOT
+IRL
+IRN
+IRQ
+ISL
+ISR
+ITA
+JAM
+JEY
+JOR
+JPN
+KAZ
+KEN
+KGZ
+KHM
+KIR
+KNA
+KOR
+KWT
+LAO
+LBN
+LBR
+LBY
+LCA
+LIE
+LKA
+LSO
+LTU
+LUX
+LVA
+MAC
+MAF
+MAR
+MCO
+MDA
+MDG
+MDV
+MEX
+MHL
+MKD
+MLI
+MLT
+MMR
+MNE
+MNG
+MNP
+MOZ
+MRT
+MSR
+MTQ
+MUS
+MWI
+MYS
+MYT
+NAM
+NCL
+NER
+NFK
+NGA
+NHB
+NIC
+NIU
+NLD
+NOR
+NPL
+NRU
+NZL
+OMN
+PAK
+PAN
+PCN
+PER
+PHL
+PLW
+PNG
+POL
+PRI
+PRK
+PRT
+PRY
+PSE
+PYF
+QAT
+REU
+ROU
+RUS
+RWA
+SAU
+SCG
+SDN
+SEN
+SGP
+SGS
+SHN
+SJM
+SLB
+SLE
+SLV
+SMR
+SOM
+SPM
+SRB
+SSD
+STP
+SUN
+SUR
+SVK
+SVN
+SWE
+SWZ
+SXM
+SYC
+SYR
+TCA
+TCD
+TGO
+THA
+TJK
+TKL
+TKM
+TLS
+TON
+TTO
+TUN
+TUR
+TUV
+TWN
+TZA
+UGA
+UKR
+UMI
+URY
+USA
+UZB
+VAT
+VCT
+VEN
+VGB
+VIR
+VNM
+VUT
+WLF
+WSM
+XKS
+YEM
+YUG
+ZAF
+ZMB
+ZWE

scripts/oecd/sdmx/download.py

@@ -0,0 +1,82 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''
+Bulk downloads OECD datasets.
+
+Note: this currently is a "best-effort" single pass and will just skip any
+datasets with errors or failures. 
+
+Produces: 
+* identifiers.csv: Series codes and names.
+* input/<CODE>.json: Folder of all fetched input datasets.
+
+Usage: OPENSSL_CONF=openssl.cnf python3 download.py
+'''
+import csv
+import json
+import requests
+import os
+from xml.etree import ElementTree
+
+PREFIX = '{http://www.SDMX.org/resources/SDMXML/schemas/v2_0/'
+
+
+def get_series():
+    '''Gets all series.
+
+    Returns:
+        Map of series code -> name.
+    '''
+    series = {}
+    response = requests.get(
+        'https://stats.oecd.org/RestSDMX/sdmx.ashx/GetKeyFamily/all')
+    root = ElementTree.fromstring(response.content)
+    for child in root:
+        for key in child:
+            if key.tag == PREFIX + 'structure}KeyFamily' and 'id' in key.attrib:
+                for attr in key:
+                    if '{http://www.w3.org/XML/1998/namespace}lang' in attr.attrib and attr.attrib[
+                            '{http://www.w3.org/XML/1998/namespace}lang'] == 'en':
+                        series[key.attrib['id']] = attr.text
+    return series
+
+
+if __name__ == '__main__':
+
+    # Precompute series list to avoid refetching on failure.
+    with open('identifiers.csv', 'w') as f:
+        series = get_series()
+        writer = csv.writer(f)
+        for s in sorted(series):
+            writer.writerow([s, series[s]])
+
+    if not os.path.exists('input'):
+        os.makedirs('input')
+
+    with open('identifiers.csv') as f_in:
+        reader = csv.reader(f_in)
+        for row in reader:
+            identifier = row[0]
+            print(identifier)
+            try:
+                result = requests.get(
+                    f'http://stats.oecd.org/sdmx-json/data/{identifier}/all/all'
+                )
+                with open(f'input/{identifier}.json', 'w') as f_out:
+                    try:
+                        f_out.write(json.dumps(result.json()))
+                    except:
+                        print('Error parsing:', identifier)
+            except:
+                print('Error fetching data for:', identifier)

scripts/oecd/sdmx/openssl.cnf

@@ -0,0 +1,10 @@
+openssl_conf = openssl_init
+
+[openssl_init]
+ssl_conf = ssl_sect
+
+[ssl_sect]
+system_default = system_default_sect
+
+[system_default_sect]
+Options = UnsafeLegacyRenegotiation