Bump urllib3 from 1.23 to 1.24.2

.gitignore

@@ -0,0 +1,10 @@
+.idea/*
+*.pyc
+*.log
+*.tar
+data/*
+logs/*
+
+*.pyc
+datim/.DS_Store
+.DS_Store

Pipfile

@@ -0,0 +1,53 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+amqp = "==2.4.2"
+astroid = "==1.6.5"
+attrs = "==19.3.0"
+billiard = "==3.5.0.5"
+celery = "==4.2.2"
+certifi = "==2018.4.16"
+chardet = "==3.0.4"
+configparser = "==3.5.0"
+contextlib2 = "==0.6.0.post1"
+deepdiff = "==3.3.0"
+dnspython = "==1.16.0"
+enum34 = "==1.1.6"
+eventlet = "==0.25.1"
+functools32 = "==3.2.3.post2"
+futures = "==3.2.0"
+gevent = "==1.4.0"
+greenlet = "==0.4.15"
+grequests = "==0.4.0"
+idna = "==2.7"
+importlib-metadata = "==1.5.0"
+isort = "==4.3.4"
+jsonpickle = "==0.9.6"
+jsonschema = "==3.2.0"
+kombu = "==4.3.0"
+lazy-object-proxy = "==1.3.1"
+mccabe = "==0.6.1"
+monotonic = "==1.5"
+ocldev = "==0.1.78"
+pathlib2 = "==2.3.5"
+pylint = "==1.9.2"
+pyrsistent = "==0.15.7"
+pytz = "==2018.9"
+redis = "==3.2.1"
+requests = "==2.24.0"
+scandir = "==1.10.0"
+singledispatch = "==3.4.0.3"
+six = "==1.11.0"
+urllib3 = "==1.24.2"
+vine = "==1.3.0"
+wrapt = "==1.10.11"
+zipp = "==1.2.0"
+"backports.functools-lru-cache" = "==1.5"
+
+[requires]
+python_version = "2.7"

README.md

@@ -1,17 +1,59 @@
+## Installation
 
-**Process**
-1) Run MER_Indicators.py - expects input file named 'MER_Indicator.csv' containing list of unique MER Indicators by uid. Results in MER_Indicator.json file
-2) Run MER_Disaggregation.py - expects input file named 'MER_Disaggregation.csv' containing list of unique MER Disaggregations by uid. Results in MER_Disaggregationr.json file
-3) Run Mechanisms.py - expects input file named 'Mechanisms partners agencies OUS Start End.csv' containing list of unique Mechanisms by uid. Results in Mechanisms.json file
+Please run `pip install -r requirements.txt` to install the required dependencies
 
+### Environment setup
+The settings for these scripts can be hard-coded or set as environment variables:
+ ```
+ dhis2env = os.environ['DHIS2_ENV'] # DHIS2 Environment URL
+ dhis2uid = os.environ['DHIS2_USER'] # DHIS2 Authentication USER
+ dhis2pwd = os.environ['DHIS2_PASS'] # DHIS2 Authentication PASSWORD
+ oclapitoken = os.environ['OCL_API_TOKEN'] # OCL Authentication API
+ oclenv = os.environ['OCL_ENV'] # DHIS2 Environment URL
+ compare2previousexport = os.environ['COMPARE_PREVIOUS_EXPORT'] in ['true', 'True']  # Whether to compare to previous export
+ ```
 
-**Changes Made**
-File: csv_to_json_flex
+ You need to specify whether you want to use the environmental variables or not and pass that as a command line argument. Example -
+ ```
+ python sims-sync.py true
+ ```
 
-1) Changed From print ('SKIPPING: '), csv_resource_def['definition_name']
-TO print 'SKIPPING: ', csv_resource_def['definition_name']
-2) Added argument and line for output JSON file
+## Scripts
+### Shared
+* settings.py
 
-FILE: MER_Indicator etc...
-1) process_by_row() function - remove 30 argument
-2) _init_ - added output_filename argument
+### IMAP
+* imapexport.py
+* imapimport.py
+* showmoh.py
+* MOH Sync Scripts
+    * syncmoh.py
+    * syncmohfy18.py
+    * syncmohfy19.py
+* Test scripts:
+    * imaptest.py
+    * imaptestcompareocl2csv.py
+    * imaptestmediator.py
+
+### QMAP
+* exportqmap.py
+* importqmap.py
+
+### Supporting scripts
+* bulkImportStatus.py
+* celeryconfig.py
+* constants.py
+* import_manager.py
+* import_util.py
+* status_util.py
+
+### Older scripts
+* showmechanisms.py
+* showmer.py
+* showsims.py
+* showtieredsupport.py
+* syncmechanisms.py
+* syncmer.py
+* syncmermsp.py
+* syncsims.py
+* synctest.py

bulkImportStatus.py

@@ -0,0 +1,75 @@
+"""
+Script to retrieve BulkImport status from OCL and return as JSON.
+Designed to work with PEPFAR IMAP and QMAP resources.
+
+python bulkImportStatus.py --env=staging -t=[my-ocl-api-token]
+    --bulkImportId=[bulk-import-task-id]
+"""
+import argparse
+import json
+import requests
+import common
+
+
+# Checks OCL bulk import status
+def check_bulk_import_status(bulkImportId='', ocl_env_url='', ocl_api_token='',
+                             import_result_format=''):
+    """ Retrieves bulk import status from OCL """
+    import_result_formats = ['report', 'json', 'summary']
+    if import_result_format not in import_result_formats:
+        import_result_format = 'summary'
+
+    ocl_api_headers = {'Content-Type': 'application/json'}
+    if ocl_api_token:
+        ocl_api_headers['Authorization'] = 'Token ' + ocl_api_token
+    import_status_url = "%s/manage/bulkimport/?task=%s&result=%s" % (
+        ocl_env_url, bulkImportId, import_result_format)
+    response = requests.get(import_status_url, headers=ocl_api_headers)
+    response.raise_for_status()
+    return response.text
+
+
+# Configure
+parser = argparse.ArgumentParser("bulkImportStatus", description="Get Bulk Import Status from OCL")
+group = parser.add_mutually_exclusive_group(required=True)
+group.add_argument('--env', help='Name of the OCL API environment', type=common.ocl_environment)
+group.add_argument('--envurl', help='URL of the OCL API environment')
+parser.add_argument('--bulkImportId', help='Bulk Import Status ID', required=True)
+parser.add_argument('-t', '--token', help='OCL API token', required=True)
+parser.add_argument(
+    '-v', '--verbosity', help='Verbosity level: 0 (default), 1, or 2', default=0, type=int)
+parser.add_argument('--version', action='version', version='%(prog)s v' + common.APP_VERSION)
+parser.add_argument(
+    '--format', help='Format of bulk import results to return from OCL', default="summary")
+parser.add_argument(
+    '--domain', help='Domain of bulk import results to return from OCL', required=True)
+args = parser.parse_args()
+ocl_env_url = args.env if args.env else args.env_url
+
+# Display debug output
+if args.verbosity > 1:
+    print args
+
+# Get the bulk import response from OCL and process -- always return as JSON response even if error
+ocl_bulk_import_response = ''
+try:
+    ocl_bulk_import_response = check_bulk_import_status(
+        bulkImportId=args.bulkImportId, ocl_env_url=ocl_env_url,
+        ocl_api_token=args.token, import_result_format=args.format)
+except Exception as e:
+    # Not Found or Error -- change the status/status_code based on response
+    output_json = {
+        "status": "Error",
+        "status_code": 0,
+        "message": ocl_bulk_import_response
+    }
+else:
+    # Pending, Completed and Success, Completed and Error during import
+    output_json = {
+        "status": "",
+        "status_code": 0,
+        "message": ocl_bulk_import_response
+    }
+
+if output_json:
+    print json.dumps(output_json)

celeryconfig.py

@@ -0,0 +1,7 @@
+""" Celery configuration """
+
+broker_url = 'redis://localhost'
+result_backend = 'redis'
+task_track_started = True
+# Results should be stored forever
+result_expires = 0

common.py

@@ -0,0 +1,75 @@
+""" Common methods and functions for command-line python tools """
+import argparse
+import requests
+
+
+# Script constants
+APP_VERSION = '0.1.0'
+OCL_ENVIRONMENTS = {
+    'qa': 'https://api.qa.openconceptlab.org',
+    'qa-v1': 'https://api.qa.v1.openconceptlab.org',
+    'qa-aws': 'https://api.qa.aws.openconceptlab.org',
+    'staging': 'https://api.staging.openconceptlab.org',
+    'staging-v1': 'https://api.staging.v1.openconceptlab.org',
+    'staging-aws': 'https://api.staging.aws.openconceptlab.org',
+    'production': 'https://api.openconceptlab.org',
+    'production-v1': 'https://api.v1.openconceptlab.org',
+    'production-aws': 'https://api.aws.openconceptlab.org',
+    'demo': 'https://api.demo.openconceptlab.org',
+    'demo-v1': 'https://api.demo.v1.openconceptlab.org',
+    'demo-aws': 'https://api.demo.aws.openconceptlab.org',
+}
+
+
+# Argument parser validation functions
+def ocl_environment(string):
+    """ Return OCL environment URL for the specified enviroment key"""
+    if string not in OCL_ENVIRONMENTS:
+        raise argparse.ArgumentTypeError('Argument "env" must be %s' % (
+            ', '.join(OCL_ENVIRONMENTS.keys())))
+    return OCL_ENVIRONMENTS[string]
+
+
+def get_imap_orgs(ocl_env_url, ocl_api_token, period_filter='', country_code_filter='',
+                  verbose=False, ocl_api_version='v2'):
+    """
+    Returns list of country Indicator Mapping organizations available in the specified OCL
+    environment. This is determined by the 'datim_moh_object' == True custom attribute of
+    the org. Orgs typically have an ID in the format 'DATIM-MOH-xx-FYyy', where 'xx' is
+    the country code (eg. CM, BI, UG) and 'yy' is the fiscal year (eg. 18, 19, 20).
+    Optional arguments 'period_filter' and 'country_code_filter' may be either a string or
+    a list and will filter the country list accordingly. For example, setting period_filter to
+    ['FY18', 'FY19'] will only return IMAP orgs from those fiscal years. Similarly, setting
+    country_code_filter to ['UG', 'BI', 'UA'] will only return those three matching
+    country codes.
+    """
+
+    # Prepare the filters
+    if period_filter:
+        if not isinstance(period_filter, list):
+            period_filter = [period_filter]
+    if country_code_filter:
+        if not isinstance(country_code_filter, list):
+            country_code_filter = [country_code_filter]
+
+    # Retrieve list of all orgs from OCL
+    extras_separator = '__' if ocl_api_version == 'v1' else '.'
+    ocl_api_headers = {'Content-Type': 'application/json'}
+    request_params = {
+        'limit': '200',
+        'verbose': 'true',
+    }
+    request_params['extras%sdatim_moh_object' % extras_separator] = 'true'
+    if period_filter:
+        request_params['extras%sdatim_moh_period' % extras_separator] = ','.join(period_filter)
+    if country_code_filter:
+        request_params['extras%sdatim_moh_country_code' % extras_separator] = ','.join(country_code_filter)
+    if ocl_api_token:
+        ocl_api_headers['Authorization'] = 'Token ' + ocl_api_token
+    url_all_orgs = '%s/orgs/' % ocl_env_url
+    response = requests.get(url_all_orgs, headers=ocl_api_headers, params=request_params)
+    if verbose:
+        print response.url
+    response.raise_for_status()
+    ocl_all_orgs = response.json()
+    return ocl_all_orgs

constants.py

@@ -0,0 +1,18 @@
+""" ocl_datim constants for celery """
+
+ENV_CELERY_CONFIG = 'celery_config'
+
+TASK_ID_KEY = 'id'
+TASK_ID_SEPARATOR = '-'
+
+RESPONSE_FIELD_ID = 'id'
+RESPONSE_FIELD_STATUS = 'status'
+RESPONSE_FIELD_RESULT = 'result'
+RESPONSE_FIELD_STATUS_CODE = 'status_code'
+
+STATUS_CODE_OK = 200
+STATUS_CODE_ACCEPTED = 202
+STATUS_CODE_BAD_REQUEST = 400
+STATUS_CODE_CONFLICT = 409
+STATUS_CODE_ERROR = 500
+STATUS_CODE_NOT_FOUND = 404