Merge pull request #41 from ianco/master

Add a check for missing relations (in addition to mis-matched relns)

scripts/bc_reg_related_companies.py

@@ -0,0 +1,154 @@
+#!/usr/bin/python
+import os
+import psycopg2
+import sys
+
+from config import (
+ get_connection,
+ get_db_sql,
+ get_sql_record_count,
+ CORP_TYPES_IN_SCOPE,
+ corp_num_with_prefix,
+ bare_corp_num,
+ is_valid_corp_num
+)
+
+
+input_companies = [
+]
+
+all_companies = []
+
+
+def get_related_companies(corp_num):
+ sql = """
+ select
+ b.identifier as firm
+ ,p.identifier as owner
+ ,p.organization_name as owner_name
+ from parties_version p
+ ,party_roles_version r
+ ,businesses b
+ where r.party_id=p.id
+ and r.business_id = b.id
+ and p.party_type='organization'
+ and r.role in ('proprietor')
+ and (b.identifier = %s or p.identifier = %s);
+ """
+
+ bc_reg_recs = get_db_sql("bc_reg_lear", sql, (corp_num, corp_num))
+ corps = []
+ for bc_reg_rec in bc_reg_recs:
+ if is_valid_corp_num(bc_reg_rec['owner']) and not (bc_reg_rec['owner'] in all_companies):
+ corps.append(bc_reg_rec['owner'])
+ all_companies.append(bc_reg_rec['owner'])
+ if is_valid_corp_num(bc_reg_rec['firm']) and not (bc_reg_rec['firm'] in all_companies):
+ corps.append(bc_reg_rec['firm'])
+ all_companies.append(bc_reg_rec['firm'])
+
+ return corps
+
+
+def get_related_companies_recursive(corp_num):
+ corps = get_related_companies(corp_num)
+ for related_corp_num in corps:
+ get_related_companies_recursive(related_corp_num)
+
+
+# mainline
+if __name__ == "__main__":
+ # start with a list of companies on the command line (index 0 is the script name)
+ companies = sys.argv[1:]
+ # if no command line companies supplied use an internal list
+ if 0 == len(companies):
+ companies = input_companies
+ for arg in companies:
+ if is_valid_corp_num(arg):
+ all_companies.append(arg)
+ get_related_companies_recursive(arg)
+
+ # build lists of corp_nums for each database
+ orgbook_corp_nums = []
+ bcreg_corp_nums = []
+ lear_corp_nums = []
+ colin_corp_nums = []
+ for corp_num in all_companies:
+ s_corp_num = "'" + corp_num + "'"
+ print(s_corp_num)
+ orgbook_corp_nums.append(s_corp_num)
+ bcreg_corp_nums.append(s_corp_num)
+ if corp_num.startswith("BC"):
+ b_corp_num = "'" + corp_num[2:] + "'"
+ bcreg_corp_nums.append(b_corp_num)
+ colin_corp_nums.append(b_corp_num)
+ elif corp_num.startswith("FM"):
+ lear_corp_nums.append(s_corp_num)
+ else:
+ colin_corp_nums.append(s_corp_num)
+
+ # print SQL's for deleting corps from OrgBook
+ delete_sqls_orgbook = {
+ 'attribute': 'delete from attribute where credential_id in (select id from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!))));',
+ 'claim': 'delete from claim where credential_id in (select id from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!))));',
+ 'name': 'delete from name where credential_id in (select id from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!))));',
+ 'credential': 'delete from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!)));',
+ 'credential_set': 'delete from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!));',
+ 'hookable_cred': 'delete from hookable_cred where corp_num in (!!corp_nums!!);',
+ 'topic_relationship': 'delete from topic_relationship where topic_id in (select id from topic where source_id in (!!corp_nums!!)) or related_topic_id in (select id from topic where source_id in (!!corp_nums!!));',
+ 'topic': 'delete from topic where source_id in (!!corp_nums!!);',
+ }
+ print(">>> OrgBook:")
+ corp_nums = ','.join(str(x) for x in orgbook_corp_nums)
+ for idx, table in enumerate(delete_sqls_orgbook):
+ x_sql = delete_sqls_orgbook[table].replace("!!corp_nums!!", corp_nums)
+ print("")
+ print(x_sql)
+ print("")
+
+ # print SQL's for queuing corps in BC Reg Isser
+ delete_sqls_bcreg = {
+ 'credential_log': 'delete from credential_log where corp_num in (!!corp_nums!!);',
+ 'corp_history_log': 'delete from corp_history_log where corp_num in (!!corp_nums!!);',
+ 'event_by_corp_filing': 'delete from event_by_corp_filing where corp_num in (!!corp_nums!!);',
+ }
+ print(">>> BC Reg Issuer:")
+ corp_nums = ','.join(str(x) for x in bcreg_corp_nums)
+ for idx, table in enumerate(delete_sqls_bcreg):
+ x_sql = delete_sqls_bcreg[table].replace("!!corp_nums!!", corp_nums)
+ print("")
+ print(x_sql)
+ print("")
+
+ insert_sqls_bcreg = {
+ 'lear': """insert into event_by_corp_filing
+(system_type_cd, corp_num, prev_event_id, prev_event_date, last_event_id, last_event_date, entry_date)
+select ebcf.system_type_cd, bc_reg_corp_num, 0, '0001-01-01', ebcf.last_event_id, ebcf.last_event_date, now()
+from event_by_corp_filing ebcf
+cross join 
+unnest(ARRAY[
+ !!corp_nums!!
+]) 
+bc_reg_corp_num
+where record_id = (select max(record_id) from event_by_corp_filing where system_type_cd = 'BCREG_LEAR');""",
+ 'colin': """insert into event_by_corp_filing
+(system_type_cd, corp_num, prev_event_id, prev_event_date, last_event_id, last_event_date, entry_date)
+select ebcf.system_type_cd, bc_reg_corp_num, 0, '0001-01-01', ebcf.last_event_id, ebcf.last_event_date, now()
+from event_by_corp_filing ebcf
+cross join 
+unnest(ARRAY[
+ !!corp_nums!!
+]) 
+bc_reg_corp_num
+where record_id = (select max(record_id) from event_by_corp_filing where system_type_cd = 'BC_REG');""",
+ }
+ print(">>> BC Reg Issuer:")
+ corp_nums = ','.join(str(x) for x in lear_corp_nums)
+ x_sql = insert_sqls_bcreg['lear'].replace("!!corp_nums!!", corp_nums)
+ print("")
+ print(x_sql)
+ corp_nums = ','.join(str(x) for x in colin_corp_nums)
+ x_sql = insert_sqls_bcreg['colin'].replace("!!corp_nums!!", corp_nums)
+ print("")
+ print(x_sql)
+ print("")
+

scripts/config.py

@@ -225,3 +225,26 @@ def bare_corp_num(corp_num):
  return corp_num[2:]
  else:
  return corp_num
+
+
+def is_valid_corp_num(corp_num):
+ if not corp_num:
+ return False
+
+ try:
+ # if corp_num is an integer add a "BC" prefix"
+ i_corp_num = int(corp_num)
+ corp_num = "BC" + corp_num
+ except:
+ pass
+
+ # all corp nums will be 8 or 9 chars
+ if 8 > len(corp_num) or 9 < len(corp_num):
+ return False
+
+ # should only be alpha-numeric
+ if not corp_num.isalnum():
+ return False
+
+ # just return True for now
+ return True

scripts/detail_audit_report.py

@@ -25,10 +25,14 @@
  get_orgbook_all_corps_csv,
  get_orgbook_missing_relations,
  get_orgbook_missing_relations_csv,
+ get_orgbook_active_relations,
+ get_orgbook_active_relations_csv,
  get_event_proc_future_corps,
  get_event_proc_future_corps_csv,
  get_bc_reg_corps, 
  get_bc_reg_corps_csv,
+ get_bc_reg_lear_all_relations,
+ get_bc_reg_lear_all_relations_csv,
 )
 from orgbook_data_audit import compare_bc_reg_orgbook
 from rocketchat_hooks import log_error, log_warning, log_info
@@ -54,9 +58,11 @@
  else:
  (orgbook_corp_types, orgbook_corp_names, orgbook_corp_infos) = get_orgbook_all_corps(USE_LEAR=USE_LEAR)
  if USE_CSV:
- orgbook_corp_relations = get_orgbook_missing_relations_csv()
+ orgbook_corp_missing_relations = get_orgbook_missing_relations_csv()
+ orgbook_corp_active_relations = get_orgbook_active_relations_csv()
  else:
- orgbook_corp_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR)
+ orgbook_corp_missing_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR)
+ orgbook_corp_active_relations = get_orgbook_active_relations(USE_LEAR=USE_LEAR)
 
  # corps that are still in the event processor queue waiting to be processed (won't be in orgbook yet)
  if USE_CSV:
@@ -67,18 +73,23 @@
  # check if all the BC Reg corps are in orgbook (with the same corp type)
  if USE_CSV:
  (bc_reg_corp_types, bc_reg_corp_names, bc_reg_corp_infos) = get_bc_reg_corps_csv()
+ (bc_reg_owners, bc_reg_firms) = get_bc_reg_lear_all_relations_csv()
  else:
  (bc_reg_corp_types, bc_reg_corp_names, bc_reg_corp_infos) = get_bc_reg_corps(USE_LEAR=USE_LEAR)
+ (bc_reg_owners, bc_reg_firms) = get_bc_reg_lear_all_relations()
 
  # do the orgbook/bc reg compare
  wrong_bus_num = compare_bc_reg_orgbook(
  bc_reg_corp_types,
  bc_reg_corp_names,
  bc_reg_corp_infos,
+ bc_reg_owners,
+ bc_reg_firms,
  orgbook_corp_types,
  orgbook_corp_names,
  orgbook_corp_infos,
- orgbook_corp_relations,
+ orgbook_corp_missing_relations,
+ orgbook_corp_active_relations,
  future_corps,
  USE_LEAR=USE_LEAR,
  )

scripts/detail_audit_report_0.py

@@ -8,7 +8,7 @@
 import csv
 
 from config import get_connection, get_db_sql, get_sql_record_count, CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num
-from orgbook_data_load import get_bc_reg_corps
+from orgbook_data_load import get_bc_reg_corps, get_bc_reg_lear_all_relations
 
 
 USE_LEAR = (os.environ.get('USE_LEAR', 'false').lower() == 'true')
@@ -21,3 +21,4 @@
  Reads all corps and corp types from the BC Reg database and writes to a csv file.
  """
  get_bc_reg_corps(USE_LEAR=USE_LEAR)
+ get_bc_reg_lear_all_relations()

scripts/detail_audit_report_2.py

@@ -9,7 +9,7 @@
 import csv
 
 from config import get_connection, get_db_sql, get_sql_record_count, CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num
-from orgbook_data_load import get_orgbook_all_corps, get_orgbook_missing_relations
+from orgbook_data_load import get_orgbook_all_corps, get_orgbook_missing_relations, get_orgbook_active_relations
 
 USE_LEAR = (os.environ.get('USE_LEAR', 'false').lower() == 'true')
 
@@ -22,4 +22,5 @@
  """
  # read from orgbook database
  (orgbook_corp_types, orgbook_corp_names, orgbook_corp_infos) = get_orgbook_all_corps(USE_LEAR=USE_LEAR)
- orgbook_corp_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR)
+ orgbook_corp_missing_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR)
+ orgbook_corp_active_relations = get_orgbook_active_relations(USE_LEAR=USE_LEAR)

scripts/orgbook_data_audit.py

@@ -86,10 +86,13 @@ def compare_bc_reg_orgbook(
  bc_reg_corp_types,
  bc_reg_corp_names,
  bc_reg_corp_infos,
+ bc_reg_owners,
+ bc_reg_firms,
  orgbook_corp_types,
  orgbook_corp_names,
  orgbook_corp_infos,
- orgbook_corp_relations,
+ orgbook_corp_missing_relations,
+ orgbook_corp_active_relations,
  future_corps,
  USE_LEAR: bool = False,
 ):
@@ -170,13 +173,15 @@ def compare_bc_reg_orgbook(
  # fixes for missing relationships
  reln_hash = {}
  reln_list = []
+ active_reln_hash = {}
+ active_reln_list = []
  if not USE_LEAR:
- for relation in orgbook_corp_relations:
+ for relation in orgbook_corp_missing_relations:
  reln = relation['s_2'] + ":" + relation['s_1']
  if not reln in reln_hash:
  reln_hash[reln] = reln
  reln_list.append(reln)
- error_msgs += "Missing relationship in OrgBook:" + reln + "\n"
+ error_msgs += "Mis-matched relationship in OrgBook:" + reln + "\n"
  reg_cmd = "queueOrgForRelnsUpdate"
  corp_num = relation['s_2']
  if corp_num.startswith('FM'):
@@ -185,14 +190,39 @@ def compare_bc_reg_orgbook(
  corp_num = corp_num[2:]
  error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['s_1'] + "\n"
 
+ for relation in orgbook_corp_active_relations:
+ source_id_1 = relation["source_id_1"]
+ source_id_2 = relation["source_id_2"]
+ o_hash = source_id_1 + ":" + source_id_2
+ active_reln_hash[o_hash] = o_hash
+ for relation in bc_reg_owners:
+ f_hash = relation["firm"] + ":" + relation["owner"]
+ o_hash = relation["owner"] + ":" + relation["firm"]
+ if (not f_hash in reln_hash) and (not f_hash in active_reln_hash):
+ active_reln_list.append(f_hash)
+ error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n"
+ reg_cmd = "queueOrgForRelnsUpdate"
+ corp_num = relation["owner"]
+ if corp_num.startswith('BC'):
+ corp_num = corp_num[2:]
+ error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n"
+ if (not f_hash in reln_hash) and (not o_hash in active_reln_hash):
+ active_reln_list.append(o_hash)
+ error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n"
+ reg_cmd = "queueOrgForRelnsUpdateLear"
+ corp_num = relation["owner"]
+ error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n"
+
  corp_errors = (len(missing_in_orgbook) +
  len(missing_in_bcreg) +
  len(wrong_corp_type) +
  len(wrong_corp_name) +
  len(wrong_corp_status) +
  len(wrong_bus_num) +
  len(wrong_corp_reg_dt) +
- len(wrong_corp_juris))
+ len(wrong_corp_juris) +
+ len(reln_list) +
+ len(active_reln_list))
 
  if 0 < corp_errors:
  log_error(error_msgs)
@@ -209,6 +239,7 @@ def compare_bc_reg_orgbook(
  error_summary += "Wrong corp jurisdiction: " + str(len(wrong_corp_juris)) + " " + str(wrong_corp_juris) + "\n"
  if not USE_LEAR:
  error_summary += "Mis-matched OrgBook relationships: " + str(len(reln_list)) + " " + str(reln_list) + "\n"
+ error_summary += "Missing OrgBook relationships: " + str(len(active_reln_list)) + " " + str(active_reln_list) + "\n"
 
  if 0 < corp_errors:
  log_error(error_summary)