From b646936653f764c1ce938f122fc252d06c25d97c Mon Sep 17 00:00:00 2001 From: Ian Costanzo Date: Mon, 26 Aug 2024 12:06:52 -0700 Subject: [PATCH 1/5] Add a check for missing relations (in addition to mis-matched rens) Signed-off-by: Ian Costanzo --- scripts/detail_audit_report.py | 17 +++- scripts/detail_audit_report_0.py | 3 +- scripts/detail_audit_report_2.py | 5 +- scripts/orgbook_data_audit.py | 39 ++++++++- scripts/orgbook_data_load.py | 141 ++++++++++++++++++++++++++++++- 5 files changed, 192 insertions(+), 13 deletions(-) diff --git a/scripts/detail_audit_report.py b/scripts/detail_audit_report.py index 11ba3e2b..b3abeba1 100644 --- a/scripts/detail_audit_report.py +++ b/scripts/detail_audit_report.py @@ -25,10 +25,14 @@ get_orgbook_all_corps_csv, get_orgbook_missing_relations, get_orgbook_missing_relations_csv, + get_orgbook_active_relations, + get_orgbook_active_relations_csv, get_event_proc_future_corps, get_event_proc_future_corps_csv, get_bc_reg_corps, get_bc_reg_corps_csv, + get_bc_reg_lear_all_relations, + get_bc_reg_lear_all_relations_csv, ) from orgbook_data_audit import compare_bc_reg_orgbook from rocketchat_hooks import log_error, log_warning, log_info @@ -54,9 +58,11 @@ else: (orgbook_corp_types, orgbook_corp_names, orgbook_corp_infos) = get_orgbook_all_corps(USE_LEAR=USE_LEAR) if USE_CSV: - orgbook_corp_relations = get_orgbook_missing_relations_csv() + orgbook_corp_missing_relations = get_orgbook_missing_relations_csv() + orgbook_corp_active_relations = get_orgbook_active_relations_csv() else: - orgbook_corp_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR) + orgbook_corp_missing_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR) + orgbook_corp_active_relations = get_orgbook_active_relations(USE_LEAR=USE_LEAR) # corps that are still in the event processor queue waiting to be processed (won't be in orgbook yet) if USE_CSV: @@ -67,18 +73,23 @@ # check if all the BC Reg corps are in orgbook (with the same corp type) if USE_CSV: (bc_reg_corp_types, bc_reg_corp_names, bc_reg_corp_infos) = get_bc_reg_corps_csv() + (bc_reg_owners, bc_reg_firms) = get_bc_reg_lear_all_relations_csv() else: (bc_reg_corp_types, bc_reg_corp_names, bc_reg_corp_infos) = get_bc_reg_corps(USE_LEAR=USE_LEAR) + (bc_reg_owners, bc_reg_firms) = get_bc_reg_lear_all_relations() # do the orgbook/bc reg compare wrong_bus_num = compare_bc_reg_orgbook( bc_reg_corp_types, bc_reg_corp_names, bc_reg_corp_infos, + bc_reg_owners, + bc_reg_firms, orgbook_corp_types, orgbook_corp_names, orgbook_corp_infos, - orgbook_corp_relations, + orgbook_corp_missing_relations, + orgbook_corp_active_relations, future_corps, USE_LEAR=USE_LEAR, ) diff --git a/scripts/detail_audit_report_0.py b/scripts/detail_audit_report_0.py index 3f9f66a9..67c8e0a0 100644 --- a/scripts/detail_audit_report_0.py +++ b/scripts/detail_audit_report_0.py @@ -8,7 +8,7 @@ import csv from config import get_connection, get_db_sql, get_sql_record_count, CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num -from orgbook_data_load import get_bc_reg_corps +from orgbook_data_load import get_bc_reg_corps, get_bc_reg_lear_all_relations USE_LEAR = (os.environ.get('USE_LEAR', 'false').lower() == 'true') @@ -21,3 +21,4 @@ Reads all corps and corp types from the BC Reg database and writes to a csv file. """ get_bc_reg_corps(USE_LEAR=USE_LEAR) + get_bc_reg_lear_all_relations() diff --git a/scripts/detail_audit_report_2.py b/scripts/detail_audit_report_2.py index edf560fb..c1cb7c5c 100644 --- a/scripts/detail_audit_report_2.py +++ b/scripts/detail_audit_report_2.py @@ -9,7 +9,7 @@ import csv from config import get_connection, get_db_sql, get_sql_record_count, CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num -from orgbook_data_load import get_orgbook_all_corps, get_orgbook_missing_relations +from orgbook_data_load import get_orgbook_all_corps, get_orgbook_missing_relations, get_orgbook_active_relations USE_LEAR = (os.environ.get('USE_LEAR', 'false').lower() == 'true') @@ -22,4 +22,5 @@ """ # read from orgbook database (orgbook_corp_types, orgbook_corp_names, orgbook_corp_infos) = get_orgbook_all_corps(USE_LEAR=USE_LEAR) - orgbook_corp_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR) + orgbook_corp_missing_relations = get_orgbook_missing_relations(USE_LEAR=USE_LEAR) + orgbook_corp_active_relations = get_orgbook_active_relations(USE_LEAR=USE_LEAR) diff --git a/scripts/orgbook_data_audit.py b/scripts/orgbook_data_audit.py index 44e73fc9..6eab5800 100644 --- a/scripts/orgbook_data_audit.py +++ b/scripts/orgbook_data_audit.py @@ -86,10 +86,13 @@ def compare_bc_reg_orgbook( bc_reg_corp_types, bc_reg_corp_names, bc_reg_corp_infos, + bc_reg_owners, + bc_reg_firms, orgbook_corp_types, orgbook_corp_names, orgbook_corp_infos, - orgbook_corp_relations, + orgbook_corp_missing_relations, + orgbook_corp_active_relations, future_corps, USE_LEAR: bool = False, ): @@ -170,13 +173,15 @@ def compare_bc_reg_orgbook( # fixes for missing relationships reln_hash = {} reln_list = [] + active_reln_hash = {} + active_reln_list = [] if not USE_LEAR: - for relation in orgbook_corp_relations: + for relation in orgbook_corp_missing_relations: reln = relation['s_2'] + ":" + relation['s_1'] if not reln in reln_hash: reln_hash[reln] = reln reln_list.append(reln) - error_msgs += "Missing relationship in OrgBook:" + reln + "\n" + error_msgs += "Mis-matched relationship in OrgBook:" + reln + "\n" reg_cmd = "queueOrgForRelnsUpdate" corp_num = relation['s_2'] if corp_num.startswith('FM'): @@ -185,6 +190,29 @@ def compare_bc_reg_orgbook( corp_num = corp_num[2:] error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['s_1'] + "\n" + for relation in orgbook_corp_active_relations: + source_id_1 = relation["source_id_1"] + source_id_2 = relation["source_id_2"] + o_hash = source_id_1 + ":" + source_id_2 + active_reln_hash[o_hash] = o_hash + for relation in bc_reg_owners: + f_hash = relation["firm"] + ":" + relation["owner"] + o_hash = relation["owner"] + ":" + relation["firm"] + if not f_hash in active_reln_hash: + active_reln_list.append(f_hash) + error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n" + reg_cmd = "queueOrgForRelnsUpdate" + corp_num = relation["owner"] + if corp_num.startswith('BC'): + corp_num = corp_num[2:] + error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n" + if not o_hash in active_reln_hash: + active_reln_list.append(o_hash) + error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n" + reg_cmd = "queueOrgForRelnsUpdateLear" + corp_num = relation["owner"] + error_cmds += "./manage -e prod " + reg_cmd + " " + relation['firm'] + " " + corp_num + "\n" + corp_errors = (len(missing_in_orgbook) + len(missing_in_bcreg) + len(wrong_corp_type) + @@ -192,7 +220,9 @@ def compare_bc_reg_orgbook( len(wrong_corp_status) + len(wrong_bus_num) + len(wrong_corp_reg_dt) + - len(wrong_corp_juris)) + len(wrong_corp_juris) + + len(reln_list) + + len(active_reln_list)) if 0 < corp_errors: log_error(error_msgs) @@ -209,6 +239,7 @@ def compare_bc_reg_orgbook( error_summary += "Wrong corp jurisdiction: " + str(len(wrong_corp_juris)) + " " + str(wrong_corp_juris) + "\n" if not USE_LEAR: error_summary += "Mis-matched OrgBook relationships: " + str(len(reln_list)) + " " + str(reln_list) + "\n" + error_summary += "Missing OrgBook relationships: " + str(len(active_reln_list)) + " " + str(active_reln_list) + "\n" if 0 < corp_errors: log_error(error_summary) diff --git a/scripts/orgbook_data_load.py b/scripts/orgbook_data_load.py index 5b8d6ec4..b761e32f 100644 --- a/scripts/orgbook_data_load.py +++ b/scripts/orgbook_data_load.py @@ -251,6 +251,73 @@ def get_bc_reg_lear_corps(): return get_bc_reg_corps_csv() +def get_bc_reg_lear_all_relations(): + """ + Reads all ACTIVE corp relationships from the BC Reg LEAR database and writes to a csv file. + """ + + # run this query against BC Reg LEAR database: + sql1 = """ + select + b.identifier as firm + ,p.identifier as owner + ,p.organization_name as owner_name + from parties p + ,party_roles r + ,businesses b + where r.party_id=p.id + and r.business_id = b.id + and p.party_type='organization' + and p.identifier is not null + and r.role in ('proprietor','partner') + and r.cessation_date is null; + """ + + bc_reg_owners = {} + bc_reg_firms = {} + bc_reg_count = 0 + with open('export/bc_reg_relations.csv', mode='w') as corp_file: + fieldnames = ["firm", "owner", "owner_name"] + corp_writer = csv.DictWriter(corp_file, fieldnames=fieldnames, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + corp_writer.writeheader() + + print("Get corp relations from BC Registries LEAR DB", datetime.datetime.now()) + start_time = time.perf_counter() + processed_count = 0 + bc_reg_recs = get_db_sql("bc_reg_lear", sql1) + for bc_reg_rec in bc_reg_recs: + bc_reg_relation = { + "owner": bc_reg_rec['owner'], + "firm": bc_reg_rec['firm'], + "owner_name": bc_reg_rec['owner_name'], + } + bc_reg_owners[bc_reg_rec['owner']] = bc_reg_relation + bc_reg_firms[bc_reg_rec['firm']] = bc_reg_relation + corp_writer.writerow(bc_reg_relation) + + return get_bc_reg_lear_all_relations_csv() + + +def get_bc_reg_lear_all_relations_csv(): + """ + Check if all the BC Reg relations are in orgbook + """ + bc_reg_owners = [] + bc_reg_firms = [] + with open('export/bc_reg_relations.csv', mode='r') as corp_file: + corp_reader = csv.DictReader(corp_file) + for row in corp_reader: + bc_reg_relation = { + "owner": row['owner'], + "firm": row['firm'], + "owner_name": row['owner_name'], + } + bc_reg_owners.append(bc_reg_relation) + bc_reg_firms.append(bc_reg_relation) + + return (bc_reg_owners, bc_reg_firms) + + def get_orgbook_all_corps(USE_LEAR: bool = False): """ Reads all companies from the orgbook database @@ -356,6 +423,74 @@ def get_orgbook_all_corps_csv(): return (orgbook_corp_types, orgbook_corp_names, orgbook_corp_infos) +def get_orgbook_active_relations(USE_LEAR: bool = False): + """ + Checks orgbook for all active relationships. + """ + conn = None + try: + conn = get_connection('org_book') + except (Exception) as error: + print(error) + raise + + # get all the mis-matched relationships from orgbook + print("Get all corp relationships from OrgBook DB", datetime.datetime.now()) + orgbook_corp_infos = {} + with open('export/orgbook_corp_active_relations.csv', mode='w') as corp_file: + fieldnames = ["rel_id", "credential_id", "source_id_1", "source_id_2", "cred_id", "effective_date", "revoked"] + corp_writer = csv.DictWriter(corp_file, fieldnames=fieldnames, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) + corp_writer.writeheader() + + if not USE_LEAR: + sql = """ + select + t_r.id rel_id, t_r.credential_id credential_id, + t_1.source_id source_id_1, t_2.source_id source_id_2, + c.id cred_id, c.effective_date effective_date, + c.revoked revoked + from topic_relationship t_r, + topic t_1, + topic t_2, + credential c + where t_r.topic_id = t_1.id + and t_r.related_topic_id = t_2.id + and c.id = t_r.credential_id + and c.revoked = false; + """ + + try: + cur = conn.cursor() + cur.execute(sql) + for row in cur: + write_corp = { + "rel_id": row[0], + "credential_id": row[1], + "source_id_1": row[2], + "source_id_2":row[3], + "cred_id": row[4], + "effective_date": row[5], + "revoked": row[6], + } + corp_writer.writerow(write_corp) + cur.close() + except (Exception) as error: + print(error) + raise + + return get_orgbook_active_relations_csv() + + +def get_orgbook_active_relations_csv(): + orgbook_corp_relations = [] + with open('export/orgbook_corp_active_relations.csv', mode='r') as corp_file: + corp_reader = csv.DictReader(corp_file) + for row in corp_reader: + orgbook_corp_relations.append(row) + + return orgbook_corp_relations + + def get_orgbook_missing_relations(USE_LEAR: bool = False): """ Checks orgbook for missing/mis-matched relationships. @@ -368,9 +503,9 @@ def get_orgbook_missing_relations(USE_LEAR: bool = False): raise # get all the mis-matched relationships from orgbook - print("Get corp relationships from OrgBook DB", datetime.datetime.now()) + print("Get mis-matched corp relationships from OrgBook DB", datetime.datetime.now()) orgbook_corp_infos = {} - with open('export/orgbook_corp_relations.csv', mode='w') as corp_file: + with open('export/orgbook_corp_missing_relations.csv', mode='w') as corp_file: fieldnames = ["tr1_topic_id", "tr1_related_topic_id", "id_1", "s_1", "id_2", "s_2"] corp_writer = csv.DictWriter(corp_file, fieldnames=fieldnames, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) corp_writer.writeheader() @@ -426,7 +561,7 @@ def get_orgbook_missing_relations(USE_LEAR: bool = False): def get_orgbook_missing_relations_csv(): orgbook_corp_relations = [] - with open('export/orgbook_corp_relations.csv', mode='r') as corp_file: + with open('export/orgbook_corp_missing_relations.csv', mode='r') as corp_file: corp_reader = csv.DictReader(corp_file) for row in corp_reader: orgbook_corp_relations.append(row) From 6ddebca7da75b2008eeca5cea50711e198518535 Mon Sep 17 00:00:00 2001 From: Ian Costanzo Date: Tue, 27 Aug 2024 10:06:05 -0700 Subject: [PATCH 2/5] Fix corp_num validation in LEAR relationships Signed-off-by: Ian Costanzo --- scripts/orgbook_data_audit.py | 4 ++-- scripts/orgbook_data_load.py | 39 +++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/scripts/orgbook_data_audit.py b/scripts/orgbook_data_audit.py index 6eab5800..d4c97a4e 100644 --- a/scripts/orgbook_data_audit.py +++ b/scripts/orgbook_data_audit.py @@ -198,7 +198,7 @@ def compare_bc_reg_orgbook( for relation in bc_reg_owners: f_hash = relation["firm"] + ":" + relation["owner"] o_hash = relation["owner"] + ":" + relation["firm"] - if not f_hash in active_reln_hash: + if (not f_hash in reln_hash) and (not f_hash in active_reln_hash): active_reln_list.append(f_hash) error_msgs += "Missing relationship in OrgBook:" + f_hash + "\n" reg_cmd = "queueOrgForRelnsUpdate" @@ -206,7 +206,7 @@ def compare_bc_reg_orgbook( if corp_num.startswith('BC'): corp_num = corp_num[2:] error_cmds += "./manage -e prod " + reg_cmd + " " + corp_num + " " + relation['firm'] + "\n" - if not o_hash in active_reln_hash: + if (not f_hash in reln_hash) and (not o_hash in active_reln_hash): active_reln_list.append(o_hash) error_msgs += "Missing relationship in OrgBook:" + o_hash + "\n" reg_cmd = "queueOrgForRelnsUpdateLear" diff --git a/scripts/orgbook_data_load.py b/scripts/orgbook_data_load.py index b761e32f..05de7745 100644 --- a/scripts/orgbook_data_load.py +++ b/scripts/orgbook_data_load.py @@ -251,6 +251,26 @@ def get_bc_reg_lear_corps(): return get_bc_reg_corps_csv() +def is_valid_corp_num(corp_num): + try: + # if corp_num is an integer add a "BC" prefix" + i_corp_num = int(corp_num) + corp_num = "BC" + corp_num + except: + pass + + # all corp nums will be 8 or 9 chars + if 8 > len(corp_num) or 9 < len(corp_num): + return False + + # should only be alpha-numeric + if not corp_num.isalnum(): + return False + + # just return True for now + return True + + def get_bc_reg_lear_all_relations(): """ Reads all ACTIVE corp relationships from the BC Reg LEAR database and writes to a csv file. @@ -269,7 +289,7 @@ def get_bc_reg_lear_all_relations(): and r.business_id = b.id and p.party_type='organization' and p.identifier is not null - and r.role in ('proprietor','partner') + and r.role in ('proprietor') and r.cessation_date is null; """ @@ -286,14 +306,15 @@ def get_bc_reg_lear_all_relations(): processed_count = 0 bc_reg_recs = get_db_sql("bc_reg_lear", sql1) for bc_reg_rec in bc_reg_recs: - bc_reg_relation = { - "owner": bc_reg_rec['owner'], - "firm": bc_reg_rec['firm'], - "owner_name": bc_reg_rec['owner_name'], - } - bc_reg_owners[bc_reg_rec['owner']] = bc_reg_relation - bc_reg_firms[bc_reg_rec['firm']] = bc_reg_relation - corp_writer.writerow(bc_reg_relation) + if is_valid_corp_num(bc_reg_rec['owner']) and is_valid_corp_num(bc_reg_rec['firm']): + bc_reg_relation = { + "owner": bc_reg_rec['owner'], + "firm": bc_reg_rec['firm'], + "owner_name": bc_reg_rec['owner_name'], + } + bc_reg_owners[bc_reg_rec['owner']] = bc_reg_relation + bc_reg_firms[bc_reg_rec['firm']] = bc_reg_relation + corp_writer.writerow(bc_reg_relation) return get_bc_reg_lear_all_relations_csv() From e2e8c3316619009d4023256e36b866bf4b35ae3f Mon Sep 17 00:00:00 2001 From: Ian Costanzo Date: Fri, 30 Aug 2024 12:40:30 -0700 Subject: [PATCH 3/5] Script to generate sql's for reprocessing companies for data fix Signed-off-by: Ian Costanzo --- scripts/bc_reg_related_companies.py | 146 ++++++++++++++++++++++++++++ scripts/config.py | 20 ++++ scripts/orgbook_data_load.py | 21 +--- 3 files changed, 167 insertions(+), 20 deletions(-) create mode 100644 scripts/bc_reg_related_companies.py diff --git a/scripts/bc_reg_related_companies.py b/scripts/bc_reg_related_companies.py new file mode 100644 index 00000000..855cf699 --- /dev/null +++ b/scripts/bc_reg_related_companies.py @@ -0,0 +1,146 @@ +#!/usr/bin/python +import os +import psycopg2 +import sys + +from config import ( + get_connection, + get_db_sql, + get_sql_record_count, + CORP_TYPES_IN_SCOPE, + corp_num_with_prefix, + bare_corp_num, + is_valid_corp_num +) + + +all_companies = [] + + +def get_related_companies(corp_num): + sql = """ + select + b.identifier as firm + ,p.identifier as owner + ,p.organization_name as owner_name + from parties_version p + ,party_roles_version r + ,businesses b + where r.party_id=p.id + and r.business_id = b.id + and p.party_type='organization' + and r.role in ('proprietor') + and (b.identifier = %s or p.identifier = %s); + """ + + bc_reg_recs = get_db_sql("bc_reg_lear", sql, (corp_num, corp_num)) + corps = [] + for bc_reg_rec in bc_reg_recs: + if is_valid_corp_num(bc_reg_rec['owner']) and not (bc_reg_rec['owner'] in all_companies): + corps.append(bc_reg_rec['owner']) + all_companies.append(bc_reg_rec['owner']) + if is_valid_corp_num(bc_reg_rec['firm']) and not (bc_reg_rec['firm'] in all_companies): + corps.append(bc_reg_rec['firm']) + all_companies.append(bc_reg_rec['firm']) + + return corps + + +def get_related_companies_recursive(corp_num): + corps = get_related_companies(corp_num) + for related_corp_num in corps: + get_related_companies_recursive(related_corp_num) + + +# mainline +if __name__ == "__main__": + # start with a list of companies on the command line (index 0 is the script name) + for arg in sys.argv[1:]: + if is_valid_corp_num(arg): + get_related_companies_recursive(arg) + + # build lists of corp_nums for each database + orgbook_corp_nums = [] + bcreg_corp_nums = [] + lear_corp_nums = [] + colin_corp_nums = [] + for corp_num in all_companies: + s_corp_num = "'" + corp_num + "'" + print(s_corp_num) + orgbook_corp_nums.append(s_corp_num) + bcreg_corp_nums.append(s_corp_num) + if corp_num.startswith("BC"): + b_corp_num = "'" + corp_num[2:] + "'" + bcreg_corp_nums.append(b_corp_num) + colin_corp_nums.append(b_corp_num) + elif corp_num.startswith("FM"): + lear_corp_nums.append(s_corp_num) + else: + colin_corp_nums.append(s_corp_num) + + # print SQL's for deleting corps from OrgBook + delete_sqls_orgbook = { + 'attribute': 'delete from attribute where credential_id in (select id from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!))));', + 'claim': 'delete from claim where credential_id in (select id from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!))));', + 'name': 'delete from name where credential_id in (select id from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!))));', + 'credential': 'delete from credential where credential_set_id in (select id from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!)));', + 'credential_set': 'delete from credential_set where topic_id in (select id from topic where source_id in (!!corp_nums!!));', + 'hookable_cred': 'delete from hookable_cred where corp_num in (!!corp_nums!!);', + 'topic_relationship': 'delete from topic_relationship where topic_id in (select id from topic where source_id in (!!corp_nums!!)) or related_topic_id in (select id from topic where source_id in (!!corp_nums!!));', + 'topic': 'delete from topic where source_id in (!!corp_nums!!);', + } + print(">>> OrgBook:") + corp_nums = ','.join(str(x) for x in orgbook_corp_nums) + for idx, table in enumerate(delete_sqls_orgbook): + x_sql = delete_sqls_orgbook[table].replace("!!corp_nums!!", corp_nums) + print("") + print(x_sql) + print("") + + # print SQL's for queuing corps in BC Reg Isser + delete_sqls_bcreg = { + 'credential_log': 'delete from credential_log where corp_num in (!!corp_nums!!);', + 'corp_history_log': 'delete from corp_history_log where corp_num in (!!corp_nums!!);', + 'event_by_corp_filing': 'delete from event_by_corp_filing where corp_num in (!!corp_nums!!);', + } + print(">>> BC Reg Issuer:") + corp_nums = ','.join(str(x) for x in bcreg_corp_nums) + for idx, table in enumerate(delete_sqls_bcreg): + x_sql = delete_sqls_bcreg[table].replace("!!corp_nums!!", corp_nums) + print("") + print(x_sql) + print("") + + insert_sqls_bcreg = { + 'lear': """insert into event_by_corp_filing +(system_type_cd, corp_num, prev_event_id, prev_event_date, last_event_id, last_event_date, entry_date) +select ebcf.system_type_cd, bc_reg_corp_num, 0, '0001-01-01', ebcf.last_event_id, ebcf.last_event_date, now() +from event_by_corp_filing ebcf +cross join +unnest(ARRAY[ + !!corp_nums!! +]) +bc_reg_corp_num +where record_id = (select max(record_id) from event_by_corp_filing where system_type_cd = 'BCREG_LEAR');""", + 'colin': """insert into event_by_corp_filing +(system_type_cd, corp_num, prev_event_id, prev_event_date, last_event_id, last_event_date, entry_date) +select ebcf.system_type_cd, bc_reg_corp_num, 0, '0001-01-01', ebcf.last_event_id, ebcf.last_event_date, now() +from event_by_corp_filing ebcf +cross join +unnest(ARRAY[ + !!corp_nums!! +]) +bc_reg_corp_num +where record_id = (select max(record_id) from event_by_corp_filing where system_type_cd = 'BC_REG');""", + } + print(">>> BC Reg Issuer:") + corp_nums = ','.join(str(x) for x in lear_corp_nums) + x_sql = insert_sqls_bcreg['lear'].replace("!!corp_nums!!", corp_nums) + print("") + print(x_sql) + corp_nums = ','.join(str(x) for x in colin_corp_nums) + x_sql = insert_sqls_bcreg['colin'].replace("!!corp_nums!!", corp_nums) + print("") + print(x_sql) + print("") + diff --git a/scripts/config.py b/scripts/config.py index 7ac2f860..15d591a8 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -225,3 +225,23 @@ def bare_corp_num(corp_num): return corp_num[2:] else: return corp_num + + +def is_valid_corp_num(corp_num): + try: + # if corp_num is an integer add a "BC" prefix" + i_corp_num = int(corp_num) + corp_num = "BC" + corp_num + except: + pass + + # all corp nums will be 8 or 9 chars + if 8 > len(corp_num) or 9 < len(corp_num): + return False + + # should only be alpha-numeric + if not corp_num.isalnum(): + return False + + # just return True for now + return True diff --git a/scripts/orgbook_data_load.py b/scripts/orgbook_data_load.py index 05de7745..6c724a48 100644 --- a/scripts/orgbook_data_load.py +++ b/scripts/orgbook_data_load.py @@ -19,6 +19,7 @@ LEAR_CORP_TYPES_IN_SCOPE, corp_num_with_prefix, bare_corp_num, + is_valid_corp_num, ) @@ -251,26 +252,6 @@ def get_bc_reg_lear_corps(): return get_bc_reg_corps_csv() -def is_valid_corp_num(corp_num): - try: - # if corp_num is an integer add a "BC" prefix" - i_corp_num = int(corp_num) - corp_num = "BC" + corp_num - except: - pass - - # all corp nums will be 8 or 9 chars - if 8 > len(corp_num) or 9 < len(corp_num): - return False - - # should only be alpha-numeric - if not corp_num.isalnum(): - return False - - # just return True for now - return True - - def get_bc_reg_lear_all_relations(): """ Reads all ACTIVE corp relationships from the BC Reg LEAR database and writes to a csv file. From 97a72c545d4f8359d92c0cb7314ceea70eee29f1 Mon Sep 17 00:00:00 2001 From: Ian Costanzo Date: Fri, 30 Aug 2024 13:02:03 -0700 Subject: [PATCH 4/5] Bug fix Signed-off-by: Ian Costanzo --- scripts/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/config.py b/scripts/config.py index 15d591a8..d9b6614e 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -228,6 +228,9 @@ def bare_corp_num(corp_num): def is_valid_corp_num(corp_num): + if not corp_num: + return False + try: # if corp_num is an integer add a "BC" prefix" i_corp_num = int(corp_num) From 29cd92bf27771280c23acaa7e6287ce6ad099af7 Mon Sep 17 00:00:00 2001 From: Ian Costanzo Date: Wed, 4 Sep 2024 14:44:20 -0700 Subject: [PATCH 5/5] Tweak script Signed-off-by: Ian Costanzo --- scripts/bc_reg_related_companies.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/bc_reg_related_companies.py b/scripts/bc_reg_related_companies.py index 855cf699..5ba33e7e 100644 --- a/scripts/bc_reg_related_companies.py +++ b/scripts/bc_reg_related_companies.py @@ -14,6 +14,9 @@ ) +input_companies = [ +] + all_companies = [] @@ -55,8 +58,13 @@ def get_related_companies_recursive(corp_num): # mainline if __name__ == "__main__": # start with a list of companies on the command line (index 0 is the script name) - for arg in sys.argv[1:]: + companies = sys.argv[1:] + # if no command line companies supplied use an internal list + if 0 == len(companies): + companies = input_companies + for arg in companies: if is_valid_corp_num(arg): + all_companies.append(arg) get_related_companies_recursive(arg) # build lists of corp_nums for each database