From de93113109a09d6d86d46739c33a40638ccda38b Mon Sep 17 00:00:00 2001 From: Ian Costanzo Date: Thu, 5 Sep 2024 10:38:32 -0700 Subject: [PATCH] Filter relationship audit based on active orgbook companies only Signed-off-by: Ian Costanzo --- scripts/orgbook_data_audit.py | 142 +++++++++++++++++++--------------- scripts/orgbook_data_load.py | 31 ++++---- 2 files changed, 94 insertions(+), 79 deletions(-) diff --git a/scripts/orgbook_data_audit.py b/scripts/orgbook_data_audit.py index d4c97a4..fcafe90 100644 --- a/scripts/orgbook_data_audit.py +++ b/scripts/orgbook_data_audit.py @@ -9,7 +9,12 @@ import requests import csv -from config import CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num +from config import ( + CORP_TYPES_IN_SCOPE, + LEAR_CORP_TYPES_IN_SCOPE, + corp_num_with_prefix, + bare_corp_num, +) from rocketchat_hooks import log_error, log_warning, log_info @@ -105,6 +110,11 @@ def compare_bc_reg_orgbook( wrong_corp_reg_dt = [] wrong_corp_juris = [] + if USE_LEAR: + corp_types_filter = LEAR_CORP_TYPES_IN_SCOPE + else: + corp_types_filter = CORP_TYPES_IN_SCOPE + # check if all the BC Reg corps are in orgbook (with the same corp type) if USE_LEAR: cmd_pfx = "Lear" @@ -116,56 +126,58 @@ def compare_bc_reg_orgbook( bc_reg_corp_type = bc_reg_corp_types[bc_reg_corp_num] bc_reg_corp_name = bc_reg_corp_names[bc_reg_corp_num] bc_reg_corp_info = bc_reg_corp_infos[bc_reg_corp_num] - if bare_corp_num(bc_reg_corp_num) in future_corps: - #print("Future corp ignore:", row["corp_num"]) - pass - elif not bc_reg_corp_num in orgbook_corp_types: - # not in orgbook - error_msgs += "Topic not found for: " + bc_reg_corp_num + "\n" - missing_in_orgbook.append(bc_reg_corp_num) - error_cmds += "./manage -e prod queueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" - pass - elif (not orgbook_corp_types[bc_reg_corp_num]) or (orgbook_corp_types[bc_reg_corp_num] != bc_reg_corp_type): - # in orgbook but has the wrong corp type in orgbook - error_msgs += "Corp Type mis-match for: " + bc_reg_corp_num + '; BC Reg: "'+bc_reg_corp_type+'", OrgBook: "'+orgbook_corp_types[bc_reg_corp_num]+'"' + "\n" - wrong_corp_type.append(bc_reg_corp_num) - error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" - error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" - elif (orgbook_corp_names[bc_reg_corp_num].strip() != bc_reg_corp_name.strip()): - # in orgbook but has the wrong corp name in orgbook - error_msgs += "Corp Name mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_name+'", OrgBook: "'+orgbook_corp_names[bc_reg_corp_num]+'"' + "\n" - wrong_corp_name.append(bc_reg_corp_num) - error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" - error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" - elif (orgbook_corp_infos[bc_reg_corp_num]["entity_status"] != bc_reg_corp_info["op_state_typ_cd"]): - # wrong entity status - error_msgs += "Corp Status mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["op_state_typ_cd"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["entity_status"]+'"' + "\n" - wrong_corp_status.append(bc_reg_corp_num) - error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" - error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" - elif (orgbook_corp_infos[bc_reg_corp_num]["bus_num"].strip() != bc_reg_corp_info["bn_9"].strip()): - # wrong BN9 business number - error_msgs += "Business Number mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["bn_9"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["bus_num"]+'"' + "\n" - wrong_bus_num.append(bc_reg_corp_num) - error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" - error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" - elif (not compare_dates(orgbook_corp_infos[bc_reg_corp_num]["registration_date"], bc_reg_corp_info["recognition_dts"], USE_LEAR=USE_LEAR)): - # wrong registration date - error_msgs += "Corp Registration Date mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["recognition_dts"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["registration_date"]+'"' + "\n" - wrong_corp_reg_dt.append(bc_reg_corp_num) - error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" - error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" - elif (orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"] != get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])): - # wrong jurisdiction - calc_juris = get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"]) - error_msgs += "Corp Jurisdiction mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+calc_juris+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"]+'"' + "\n" - wrong_corp_juris.append(bc_reg_corp_num) - error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" - error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" + if bc_reg_corp_type in corp_types_filter: + if bare_corp_num(bc_reg_corp_num) in future_corps: + #print("Future corp ignore:", row["corp_num"]) + pass + elif not bc_reg_corp_num in orgbook_corp_types: + # not in orgbook + error_msgs += "Topic not found for: " + bc_reg_corp_num + "\n" + missing_in_orgbook.append(bc_reg_corp_num) + error_cmds += "./manage -e prod queueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" + pass + elif (not orgbook_corp_types[bc_reg_corp_num]) or (orgbook_corp_types[bc_reg_corp_num] != bc_reg_corp_type): + # in orgbook but has the wrong corp type in orgbook + error_msgs += "Corp Type mis-match for: " + bc_reg_corp_num + '; BC Reg: "'+bc_reg_corp_type+'", OrgBook: "'+orgbook_corp_types[bc_reg_corp_num]+'"' + "\n" + wrong_corp_type.append(bc_reg_corp_num) + error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" + error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" + elif (orgbook_corp_names[bc_reg_corp_num].strip() != bc_reg_corp_name.strip()): + # in orgbook but has the wrong corp name in orgbook + error_msgs += "Corp Name mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_name+'", OrgBook: "'+orgbook_corp_names[bc_reg_corp_num]+'"' + "\n" + wrong_corp_name.append(bc_reg_corp_num) + error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" + error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" + elif (orgbook_corp_infos[bc_reg_corp_num]["entity_status"] != bc_reg_corp_info["op_state_typ_cd"]): + # wrong entity status + error_msgs += "Corp Status mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["op_state_typ_cd"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["entity_status"]+'"' + "\n" + wrong_corp_status.append(bc_reg_corp_num) + error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" + error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" + elif (orgbook_corp_infos[bc_reg_corp_num]["bus_num"].strip() != bc_reg_corp_info["bn_9"].strip()): + # wrong BN9 business number + error_msgs += "Business Number mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["bn_9"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["bus_num"]+'"' + "\n" + wrong_bus_num.append(bc_reg_corp_num) + error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" + error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" + elif (not compare_dates(orgbook_corp_infos[bc_reg_corp_num]["registration_date"], bc_reg_corp_info["recognition_dts"], USE_LEAR=USE_LEAR)): + # wrong registration date + error_msgs += "Corp Registration Date mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["recognition_dts"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["registration_date"]+'"' + "\n" + wrong_corp_reg_dt.append(bc_reg_corp_num) + error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" + error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" + elif (orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"] != get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])): + # wrong jurisdiction + calc_juris = get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"]) + error_msgs += "Corp Jurisdiction mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+calc_juris+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"]+'"' + "\n" + wrong_corp_juris.append(bc_reg_corp_num) + error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n" + error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n" # now check if there are corps in orgbook that are *not* in BC Reg database for orgbook_corp in orgbook_corp_types: - if not (orgbook_corp in bc_reg_corp_types): + bc_reg_corp_type = bc_reg_corp_types.get(orgbook_corp) + if (bc_reg_corp_type and bc_reg_corp_type in corp_types_filter) and not (orgbook_corp in bc_reg_corp_types): missing_in_bcreg.append(orgbook_corp) error_msgs += "OrgBook corp not in BC Reg: " + orgbook_corp + "\n" error_cmds += "./manage -p bc -e prod deleteTopic " + orgbook_corp + "\n" @@ -196,22 +208,24 @@ def compare_bc_reg_orgbook( o_hash = source_id_1 + ":" + source_id_2 active_reln_hash[o_hash] = o_hash for relation in bc_reg_owners: - f_hash = relation["firm"] + ":" + relation["owner"] - o_hash = relation["owner"] + ":" + relation["firm"] - if (not f_hash in reln_hash) and (not f_hash in active_reln_hash): - active_reln_list.append(f_hash) - error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n" - reg_cmd = "queueOrgForRelnsUpdate" - corp_num = relation["owner"] - if corp_num.startswith('BC'): - corp_num = corp_num[2:] - error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n" - if (not f_hash in reln_hash) and (not o_hash in active_reln_hash): - active_reln_list.append(o_hash) - error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n" - reg_cmd = "queueOrgForRelnsUpdateLear" - corp_num = relation["owner"] - error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n" + # only check if both firms are already in OrgBook + if relation["firm"] in bc_reg_corp_types and relation["owner"] in bc_reg_corp_types: + f_hash = relation["firm"] + ":" + relation["owner"] + o_hash = relation["owner"] + ":" + relation["firm"] + if (not f_hash in reln_hash) and (not f_hash in active_reln_hash): + active_reln_list.append(f_hash) + error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n" + reg_cmd = "queueOrgForRelnsUpdate" + corp_num = relation["owner"] + if corp_num.startswith('BC'): + corp_num = corp_num[2:] + error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n" + if (not f_hash in reln_hash) and (not o_hash in active_reln_hash): + active_reln_list.append(o_hash) + error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n" + reg_cmd = "queueOrgForRelnsUpdateLear" + corp_num = relation["owner"] + error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n" corp_errors = (len(missing_in_orgbook) + len(missing_in_bcreg) + diff --git a/scripts/orgbook_data_load.py b/scripts/orgbook_data_load.py index 6c724a4..da7174e 100644 --- a/scripts/orgbook_data_load.py +++ b/scripts/orgbook_data_load.py @@ -388,21 +388,22 @@ def get_orgbook_all_corps(USE_LEAR: bool = False): cur.execute(sql4) for row in cur: # row[1] is the corp_type - if row[1] in corp_types_filter: - orgbook_corp_types[row[0]] = row[1] - corp_name = row[4] if (row[4] and 0 < len(row[4])) else row[3] - orgbook_corp_names[row[0]] = corp_name - write_corp = { - "corp_num": row[0], - "corp_type": row[1], - "registration_date": row[2], - "corp_name":corp_name, - "home_jurisdiction": row[5], - "entity_status": row[6], - "bus_num": row[7], - } - corp_writer.writerow(write_corp) - orgbook_corp_infos[row[0]] = write_corp + # if row[1] in corp_types_filter: + # load all orgs and check the filter when running the audit report + orgbook_corp_types[row[0]] = row[1] + corp_name = row[4] if (row[4] and 0 < len(row[4])) else row[3] + orgbook_corp_names[row[0]] = corp_name + write_corp = { + "corp_num": row[0], + "corp_type": row[1], + "registration_date": row[2], + "corp_name":corp_name, + "home_jurisdiction": row[5], + "entity_status": row[6], + "bus_num": row[7], + } + corp_writer.writerow(write_corp) + orgbook_corp_infos[row[0]] = write_corp cur.close() except (Exception) as error: print(error)