world_bank_datasets

scripts/world_bank/datasets/README.md

@@ -0,0 +1,66 @@
+# World Bank Datasets
+
+- source: https://data.worldbank.org
+
+- how to download data: Auto download of data by using python script(datasets.py).
+
+- type of place: Country.
+
+- statvars: All Type
+
+- years: 1960 to 2050
+
+-copyright year: 2024
+
+### How to run:
+"""Processes WB datasets.
+
+update september 2024:
+To run all processing methods , please do not pass the mode 
+Run: python3 datasets.py
+
+Or If required to check issue in any individual process follow all the steps as below:
+
+Supports the following tasks:
+
+============================
+
+fetch_datasets: Fetches WB dataset lists and resources and writes them to 'output/wb-datasets.csv'
+
+Run: python3 datasets.py --mode=fetch_datasets
+
+============================
+
+download_datasets: Downloads datasets listed in 'output/wb-datasets.csv' to the 'output/downloads' folder.
+
+Run: python3 datasets.py --mode=download_datasets
+
+============================
+
+write_wb_codes: Extracts World Bank indicator codes (and related information) from files downloaded in the 'output/downloads' folder to 'output/wb-codes.csv'.
+
+It only operates on files that are named '*_CSV.zip'.
+
+Run: python3 datasets.py --mode=write_wb_codes
+
+============================
+
+load_stat_vars: Loads stat vars from a mapping file specified via the `stat_vars_file` flag.
+
+Use this for debugging to ensure that the mappings load correctly and fix any errors logged by this operation.
+
+Run: python3 datasets.py --mode=load_stat_vars --stat_vars_file=/path/to/statvars.csv
+
+See `sample-svs.csv` for a sample mappings file.
+
+============================
+
+write_observations: Extracts observations from files downloaded in the 'output/downloads' folder and saves them to CSVs in the 'output/observations' folder.
+
+The stat vars file to be used for mappings should be specified using the `stat_vars_file' flag.
+
+It only operates on files that are named '*_CSV.zip'.
+
+Run: python3 datasets.py --mode=write_observations --stat_vars_file=/path/to/statvars.csv
+"""
+

scripts/world_bank/datasets/datasets.py

@@ -13,6 +13,12 @@
 # limitations under the License.
 """Processes WB datasets.
 
+update september 2024:
+To run all processing methods , please do not pass the mode 
+Run: python3 datasets.py
+
+Or If required to check issue in any individual process follow all the steps as below:
+
 Supports the following tasks:
 
 ============================
@@ -41,7 +47,7 @@
 
 Use this for debugging to ensure that the mappings load correctly and fix any errors logged by this operation.
 
-Run: python3 datasets.py --mode=load_stat_vars --stat_vars_file=/path/to/sv_mappings.csv
+Run: python3 datasets.py --mode=load_stat_vars --stat_vars_file=/path/to/statvars.csv
 
 See `sample-svs.csv` for a sample mappings file.
 
@@ -53,7 +59,7 @@
 
 It only operates on files that are named '*_CSV.zip'.
 
-Run: python3 datasets.py --mode=write_observations --stat_vars_file=/path/to/sv_mappings.csv
+Run: python3 datasets.py --mode=write_observations --stat_vars_file=/path/to/statvars.csv
 """
 
 import requests
@@ -66,6 +72,7 @@
 import re
 import urllib3
 from urllib3.util.ssl_ import create_urllib3_context
+from urllib3.exceptions import HTTPError
 from absl import flags
 import zipfile
 import codecs
@@ -84,7 +91,7 @@ class Mode:
 
 
 flags.DEFINE_string(
- 'mode', Mode.WRITE_OBSERVATIONS,
+ 'mode', None,
  f"Specify one of the following modes: {Mode.FETCH_DATASETS}, {Mode.DOWNLOAD_DATASETS}, {Mode.WRITE_WB_CODES}, {Mode.LOAD_STAT_VARS}, {Mode.WRITE_OBSERVATIONS}"
 )
 
@@ -131,7 +138,7 @@ class Mode:
 
 def download_datasets():
  '''Downloads dataset files. This is a very expensive operation so run it with care. It assumes that the datasets CSV is already available.'''
-
+ logging.info('start download_datasets')
  with open(DATASETS_CSV_FILE_PATH, 'r') as f:
  csv_rows = list(csv.DictReader(f))
  download_urls = []
@@ -158,10 +165,13 @@ def download(url):
  # response = requests.get(url)
  # Using urllib3 for downloading content to avoid SSL issue.
  # See: https://github.com/urllib3/urllib3/issues/2653#issuecomment-1165418616
- with urllib3.PoolManager(ssl_context=ctx) as http:
- response = http.request("GET", url)
- with open(file_path, 'wb') as f:
- f.write(response.data)
+ with urllib3.PoolManager(ssl_context=ctx,timeout=90) as http:
+ try:
+ response = http.request("GET", url)
+ with open(file_path, 'wb') as f:
+ f.write(response.data)
+ except HTTPError as e:
+ print(f"HTTP error encountered: {e}")
  except Exception as e:
  logging.error("Error downloading %s", url, exc_info=e)
 
@@ -277,11 +287,15 @@ def load_json(url, params, response_file):
  return json.load(f)
 
  logging.info("Fetching url %s, params %s", url, params)
- response = requests.get(url, params=params).json()
- with open(response_file, 'w') as f:
- logging.info('Writing response to file %s', response_file)
- json.dump(response, f, indent=2)
- return response
+ try:
+ response = requests.get(url, params=params).json()
+ with open(response_file, 'w') as f:
+ logging.info('Writing response to file %s', response_file)
+ json.dump(response, f, indent=2)
+ return response
+ except Exception as e:
+ print(f"Http error {e}")
+ return None
 
 
 def load_json_file(json_file):
@@ -571,19 +585,27 @@ def get_data_and_series_file_names(zip):
 
 
 def main(_):
- match FLAGS.mode:
- case Mode.FETCH_DATASETS:
- download_datasets()
- case Mode.DOWNLOAD_DATASETS:
- fetch_and_write_datasets_csv()
- case Mode.WRITE_WB_CODES:
- write_wb_codes()
- case Mode.LOAD_STAT_VARS:
- load_stat_vars(FLAGS.stat_vars_file)
- case Mode.WRITE_OBSERVATIONS:
- write_all_observations(FLAGS.stat_vars_file)
- case _:
- logging.error('No mode specified.')
+ logging.info(FLAGS.mode)
+ if not FLAGS.mode:
+ fetch_and_write_datasets_csv()
+ download_datasets()
+ write_wb_codes()
+ load_stat_vars(FLAGS.stat_vars_file)
+ write_all_observations(FLAGS.stat_vars_file)
+ else:
+ match FLAGS.mode:
+ case Mode.FETCH_DATASETS:
+ download_datasets()
+ case Mode.DOWNLOAD_DATASETS:
+ fetch_and_write_datasets_csv()
+ case Mode.WRITE_WB_CODES:
+ write_wb_codes()
+ case Mode.LOAD_STAT_VARS:
+ load_stat_vars(FLAGS.stat_vars_file)
+ case Mode.WRITE_OBSERVATIONS:
+ write_all_observations(FLAGS.stat_vars_file)
+ case _:
+ logging.error('No mode specified.')
 
 
 if __name__ == '__main__':

scripts/world_bank/datasets/manifest.json

@@ -0,0 +1,94 @@
+{
+ "import_specifications": [
+ {
+ "import_name": "WorldBankDatasets",
+ "curator_emails": ["[email protected]"],
+ "provenance_url": "https://data.worldbank.org",
+ "provenance_description": "World Bank databases are essential tools for supporting critical management decisions and providing key statistical information for Bank operational activities.",
+ "scripts": ["datasets.py"],
+ "import_inputs": [
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/ASPIRE_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/EdStats_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/FINDEX_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/GFDD_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/GPFI_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/HCI_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/IDA_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/Jobs_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/MDG_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/PovStats_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/SDG_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/SE4ALL_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/Subnational-Population_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/Subnational-Poverty_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/WGI_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/BBSC_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/DB_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/Economic_Fitness_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/HEFPI_CSV_obs.csv"
+ },
+ {
+ "template_mcf": "wb.tmcf",
+ "cleaned_csv": "output/observations/WWBI_CSV_obs.csv"
+ }
+ ],
+ "cron_schedule": "5 3 15 * *"
+ }
+ ]
+}