Merge pull request #43 from ianco/master

Filter relationship audit based on active orgbook companies only
bcgov · Sep 5, 2024 · 51faabf · 51faabf
2 parents 845cdcc + de93113
commit 51faabf
Show file tree

Hide file tree

Showing 2 changed files with 94 additions and 79 deletions.
diff --git a/scripts/orgbook_data_audit.py b/scripts/orgbook_data_audit.py
@@ -9,7 +9,12 @@
 import requests
 import csv
 
-from config import CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num
+from config import (
+ CORP_TYPES_IN_SCOPE,
+ LEAR_CORP_TYPES_IN_SCOPE,
+ corp_num_with_prefix,
+ bare_corp_num,
+)
 from rocketchat_hooks import log_error, log_warning, log_info
 
 
@@ -105,6 +110,11 @@ def compare_bc_reg_orgbook(
  wrong_corp_reg_dt = []
  wrong_corp_juris = []
 
+ if USE_LEAR:
+ corp_types_filter = LEAR_CORP_TYPES_IN_SCOPE
+ else:
+ corp_types_filter = CORP_TYPES_IN_SCOPE
+
  # check if all the BC Reg corps are in orgbook (with the same corp type)
  if USE_LEAR:
  cmd_pfx = "Lear"
@@ -116,56 +126,58 @@ def compare_bc_reg_orgbook(
  bc_reg_corp_type = bc_reg_corp_types[bc_reg_corp_num]
  bc_reg_corp_name = bc_reg_corp_names[bc_reg_corp_num]
  bc_reg_corp_info = bc_reg_corp_infos[bc_reg_corp_num]
- if bare_corp_num(bc_reg_corp_num) in future_corps:
- #print("Future corp ignore:", row["corp_num"])
- pass
- elif not bc_reg_corp_num in orgbook_corp_types:
- # not in orgbook
- error_msgs += "Topic not found for: " + bc_reg_corp_num + "\n"
- missing_in_orgbook.append(bc_reg_corp_num)
- error_cmds += "./manage -e prod queueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
- pass
- elif (not orgbook_corp_types[bc_reg_corp_num]) or (orgbook_corp_types[bc_reg_corp_num] != bc_reg_corp_type):
- # in orgbook but has the wrong corp type in orgbook
- error_msgs += "Corp Type mis-match for: " + bc_reg_corp_num + '; BC Reg: "'+bc_reg_corp_type+'", OrgBook: "'+orgbook_corp_types[bc_reg_corp_num]+'"' + "\n"
- wrong_corp_type.append(bc_reg_corp_num)
- error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
- error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
- elif (orgbook_corp_names[bc_reg_corp_num].strip() != bc_reg_corp_name.strip()):
- # in orgbook but has the wrong corp name in orgbook
- error_msgs += "Corp Name mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_name+'", OrgBook: "'+orgbook_corp_names[bc_reg_corp_num]+'"' + "\n"
- wrong_corp_name.append(bc_reg_corp_num)
- error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
- error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
- elif (orgbook_corp_infos[bc_reg_corp_num]["entity_status"] != bc_reg_corp_info["op_state_typ_cd"]):
- # wrong entity status
- error_msgs += "Corp Status mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["op_state_typ_cd"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["entity_status"]+'"' + "\n"
- wrong_corp_status.append(bc_reg_corp_num)
- error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
- error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
- elif (orgbook_corp_infos[bc_reg_corp_num]["bus_num"].strip() != bc_reg_corp_info["bn_9"].strip()):
- # wrong BN9 business number
- error_msgs += "Business Number mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["bn_9"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["bus_num"]+'"' + "\n"
- wrong_bus_num.append(bc_reg_corp_num)
- error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
- error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
- elif (not compare_dates(orgbook_corp_infos[bc_reg_corp_num]["registration_date"], bc_reg_corp_info["recognition_dts"], USE_LEAR=USE_LEAR)):
- # wrong registration date
- error_msgs += "Corp Registration Date mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["recognition_dts"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["registration_date"]+'"' + "\n"
- wrong_corp_reg_dt.append(bc_reg_corp_num)
- error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
- error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
- elif (orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"] != get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])):
- # wrong jurisdiction
- calc_juris = get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])
- error_msgs += "Corp Jurisdiction mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+calc_juris+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"]+'"' + "\n"
- wrong_corp_juris.append(bc_reg_corp_num)
- error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
- error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
+ if bc_reg_corp_type in corp_types_filter:
+ if bare_corp_num(bc_reg_corp_num) in future_corps:
+ #print("Future corp ignore:", row["corp_num"])
+ pass
+ elif not bc_reg_corp_num in orgbook_corp_types:
+ # not in orgbook
+ error_msgs += "Topic not found for: " + bc_reg_corp_num + "\n"
+ missing_in_orgbook.append(bc_reg_corp_num)
+ error_cmds += "./manage -e prod queueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
+ pass
+ elif (not orgbook_corp_types[bc_reg_corp_num]) or (orgbook_corp_types[bc_reg_corp_num] != bc_reg_corp_type):
+ # in orgbook but has the wrong corp type in orgbook
+ error_msgs += "Corp Type mis-match for: " + bc_reg_corp_num + '; BC Reg: "'+bc_reg_corp_type+'", OrgBook: "'+orgbook_corp_types[bc_reg_corp_num]+'"' + "\n"
+ wrong_corp_type.append(bc_reg_corp_num)
+ error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
+ error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
+ elif (orgbook_corp_names[bc_reg_corp_num].strip() != bc_reg_corp_name.strip()):
+ # in orgbook but has the wrong corp name in orgbook
+ error_msgs += "Corp Name mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_name+'", OrgBook: "'+orgbook_corp_names[bc_reg_corp_num]+'"' + "\n"
+ wrong_corp_name.append(bc_reg_corp_num)
+ error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
+ error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
+ elif (orgbook_corp_infos[bc_reg_corp_num]["entity_status"] != bc_reg_corp_info["op_state_typ_cd"]):
+ # wrong entity status
+ error_msgs += "Corp Status mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["op_state_typ_cd"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["entity_status"]+'"' + "\n"
+ wrong_corp_status.append(bc_reg_corp_num)
+ error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
+ error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
+ elif (orgbook_corp_infos[bc_reg_corp_num]["bus_num"].strip() != bc_reg_corp_info["bn_9"].strip()):
+ # wrong BN9 business number
+ error_msgs += "Business Number mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["bn_9"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["bus_num"]+'"' + "\n"
+ wrong_bus_num.append(bc_reg_corp_num)
+ error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
+ error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
+ elif (not compare_dates(orgbook_corp_infos[bc_reg_corp_num]["registration_date"], bc_reg_corp_info["recognition_dts"], USE_LEAR=USE_LEAR)):
+ # wrong registration date
+ error_msgs += "Corp Registration Date mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+bc_reg_corp_info["recognition_dts"]+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["registration_date"]+'"' + "\n"
+ wrong_corp_reg_dt.append(bc_reg_corp_num)
+ error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
+ error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
+ elif (orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"] != get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])):
+ # wrong jurisdiction
+ calc_juris = get_corp_jurisdiction(bc_reg_corp_info["corp_type"], bc_reg_corp_info["corp_class"], bc_reg_corp_info["can_jur_typ_cd"], bc_reg_corp_info["othr_juris_desc"])
+ error_msgs += "Corp Jurisdiction mis-match for: " + bc_reg_corp_num + ' BC Reg: "'+calc_juris+'", OrgBook: "'+orgbook_corp_infos[bc_reg_corp_num]["home_jurisdiction"]+'"' + "\n"
+ wrong_corp_juris.append(bc_reg_corp_num)
+ error_cmds += "./manage -p bc -e prod deleteTopic " + bc_reg_corp_num + "\n"
+ error_cmds += "./manage -e prod requeueOrganization" + cmd_pfx + " " + bare_corp_num(bc_reg_corp_num) + "\n"
 
  # now check if there are corps in orgbook that are *not* in BC Reg database
  for orgbook_corp in orgbook_corp_types:
- if not (orgbook_corp in bc_reg_corp_types):
+ bc_reg_corp_type = bc_reg_corp_types.get(orgbook_corp)
+ if (bc_reg_corp_type and bc_reg_corp_type in corp_types_filter) and not (orgbook_corp in bc_reg_corp_types):
  missing_in_bcreg.append(orgbook_corp)
  error_msgs += "OrgBook corp not in BC Reg: " + orgbook_corp + "\n"
  error_cmds += "./manage -p bc -e prod deleteTopic " + orgbook_corp + "\n"
@@ -196,22 +208,24 @@ def compare_bc_reg_orgbook(
  o_hash = source_id_1 + ":" + source_id_2
  active_reln_hash[o_hash] = o_hash
  for relation in bc_reg_owners:
- f_hash = relation["firm"] + ":" + relation["owner"]
- o_hash = relation["owner"] + ":" + relation["firm"]
- if (not f_hash in reln_hash) and (not f_hash in active_reln_hash):
- active_reln_list.append(f_hash)
- error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n"
- reg_cmd = "queueOrgForRelnsUpdate"
- corp_num = relation["owner"]
- if corp_num.startswith('BC'):
- corp_num = corp_num[2:]
- error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n"
- if (not f_hash in reln_hash) and (not o_hash in active_reln_hash):
- active_reln_list.append(o_hash)
- error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n"
- reg_cmd = "queueOrgForRelnsUpdateLear"
- corp_num = relation["owner"]
- error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n"
+ # only check if both firms are already in OrgBook
+ if relation["firm"] in bc_reg_corp_types and relation["owner"] in bc_reg_corp_types:
+ f_hash = relation["firm"] + ":" + relation["owner"]
+ o_hash = relation["owner"] + ":" + relation["firm"]
+ if (not f_hash in reln_hash) and (not f_hash in active_reln_hash):
+ active_reln_list.append(f_hash)
+ error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n"
+ reg_cmd = "queueOrgForRelnsUpdate"
+ corp_num = relation["owner"]
+ if corp_num.startswith('BC'):
+ corp_num = corp_num[2:]
+ error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n"
+ if (not f_hash in reln_hash) and (not o_hash in active_reln_hash):
+ active_reln_list.append(o_hash)
+ error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n"
+ reg_cmd = "queueOrgForRelnsUpdateLear"
+ corp_num = relation["owner"]
+ error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n"
 
  corp_errors = (len(missing_in_orgbook) +
  len(missing_in_bcreg) +

diff --git a/scripts/orgbook_data_load.py b/scripts/orgbook_data_load.py
@@ -388,21 +388,22 @@ def get_orgbook_all_corps(USE_LEAR: bool = False):
  cur.execute(sql4)
  for row in cur:
  # row[1] is the corp_type
- if row[1] in corp_types_filter:
- orgbook_corp_types[row[0]] = row[1]
- corp_name = row[4] if (row[4] and 0 < len(row[4])) else row[3]
- orgbook_corp_names[row[0]] = corp_name
- write_corp = {
- "corp_num": row[0],
- "corp_type": row[1],
- "registration_date": row[2],
- "corp_name":corp_name,
- "home_jurisdiction": row[5],
- "entity_status": row[6],
- "bus_num": row[7],
- }
- corp_writer.writerow(write_corp)
- orgbook_corp_infos[row[0]] = write_corp
+ # if row[1] in corp_types_filter:
+ # load all orgs and check the filter when running the audit report
+ orgbook_corp_types[row[0]] = row[1]
+ corp_name = row[4] if (row[4] and 0 < len(row[4])) else row[3]
+ orgbook_corp_names[row[0]] = corp_name
+ write_corp = {
+ "corp_num": row[0],
+ "corp_type": row[1],
+ "registration_date": row[2],
+ "corp_name":corp_name,
+ "home_jurisdiction": row[5],
+ "entity_status": row[6],
+ "bus_num": row[7],
+ }
+ corp_writer.writerow(write_corp)
+ orgbook_corp_infos[row[0]] = write_corp
  cur.close()
  except (Exception) as error:
  print(error)