From 6ed388862432d05bf9d77180047b67fc3f6c8b3a Mon Sep 17 00:00:00 2001 From: Open Risk Date: Mon, 3 Apr 2023 16:56:15 +0200 Subject: [PATCH] v0.6.1 (full sflp load) --- CHANGELOG.rst | 6 +- sflp_portfolio/admin.py | 31 +- .../management/commands/load_core_sflp_csv.py | 10 +- .../management/commands/load_full_sflp_csv.py | 512 +++++++++++------- .../commands/load_static_sflp_csv.py | 6 +- sflp_portfolio/models/counterparty.py | 6 +- sflp_portfolio/models/counterparty_state.py | 4 +- sflp_portfolio/models/enforcement.py | 9 +- sflp_portfolio/models/forbearance.py | 15 +- sflp_portfolio/models/loan.py | 5 + sflp_portfolio/models/loan_state.py | 33 +- sflp_portfolio/models/property_collateral.py | 4 +- .../models/property_collateral_state.py | 2 +- 13 files changed, 377 insertions(+), 266 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 30095df..8e7dec2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,9 +2,13 @@ ChangeLog =========================== PLEASE NOTE THIS IS ONLY A BETA RELEASE. THE OPENNPL API IS STILL UNSTABLE +v0.6.1 (03-04-2023) +------------------- +* Functionality: Complete SFLP Template Loading + v0.6.0 (13-03-2023) ------------------- -* Functionality: Almost complete Single Family Loan Performance Templates (SFLP) (except repayment strings) +* Functionality: Almost complete Single Family Loan Performance Templates (SFLP) (except forbearance, enforcement and repayment strings) * Documentation: SFLP Data Models v0.5.3 (18-01-2023) diff --git a/sflp_portfolio/admin.py b/sflp_portfolio/admin.py index 1de7614..25e7889 100755 --- a/sflp_portfolio/admin.py +++ b/sflp_portfolio/admin.py @@ -44,62 +44,69 @@ class PortfolioAdmin(admin.ModelAdmin): save_as = True view_on_site = False # search_fields = ['name'] - # list_display = ('name', 'creation_date', 'last_change_date') + list_display = ('name', 'creation_date', 'last_change_date') class Portfolio_SnapshotAdmin(admin.ModelAdmin): save_as = True view_on_site = False - # search_fields = ['name'] - # list_display = ('name', 'creation_date', 'cutoff_date') + # search_fields = ['monthly_reporting_period'] + list_display = ('monthly_reporting_period',) class CounterpartyAdmin(admin.ModelAdmin): save_as = True view_on_site = False + list_display = ('counterparty_identifier', 'loan_identifier', 'debt_to_income', 'number_of_borrowers', + 'borrower_credit_score_at_origination') class CounterpartyStateAdmin(admin.ModelAdmin): save_as = True view_on_site = False + # TODO date_hierarchy = ('portfolio_snapshot_id') + list_display = ('counterparty_identifier', 'portfolio_snapshot_id') class LoanAdmin(admin.ModelAdmin): save_as = True view_on_site = False - search_fields = ['loan_identifier'] + list_display = ( + 'loan_identifier', 'channel', 'original_loan_term', 'original_upb', 'original_loan_to_value_ratio', + 'loan_purpose') class LoanStateAdmin(admin.ModelAdmin): save_as = True view_on_site = False - list_display = ('loan_id', 'portfolio_snapshot_id') + list_display = ('loan_identifier', 'portfolio_snapshot_id', 'servicer_name', 'total_principal_current', + 'remaining_months_to_legal_maturity') class PropertyCollateralAdmin(admin.ModelAdmin): save_as = True view_on_site = False + list_display = ( + 'property_type', 'number_of_units', 'metropolitan_statistical_area', 'property_state', 'zip_code_short') class PropertyCollateralStateAdmin(admin.ModelAdmin): save_as = True view_on_site = False - - -class ExternalCollectionAdmin(admin.ModelAdmin): - save_as = True - view_on_site = False + list_display = ('property_collateral_id', 'portfolio_snapshot_id', 'property_valuation_method') class EnforcementAdmin(admin.ModelAdmin): save_as = True view_on_site = False - + list_display = ('property_collateral_identifier', 'portfolio_snapshot_id', + 'net_sales_proceeds', 'asset_recovery_costs', 'foreclosure_costs') class ForbearanceAdmin(admin.ModelAdmin): save_as = True view_on_site = False - + list_display = ('loan_identifier', 'portfolio_snapshot_id', 'modification_flag', + 'noninterest_bearing_upb', 'principal_forgiveness_amount') admin.site.register(Portfolio, PortfolioAdmin) diff --git a/sflp_portfolio/management/commands/load_core_sflp_csv.py b/sflp_portfolio/management/commands/load_core_sflp_csv.py index 321c86c..dbeeb22 100644 --- a/sflp_portfolio/management/commands/load_core_sflp_csv.py +++ b/sflp_portfolio/management/commands/load_core_sflp_csv.py @@ -145,7 +145,7 @@ class Command(BaseCommand): # print('LoanState: ', i) loan_state = LoanState( id=i, - loan_id=loan_dict[entry[0]], + loan_identifier=loan_dict[entry[0]], portfolio_snapshot_id=portfolio_snapshot_dict[entry[1]], high_loan_to_value_refinance_option_indicator=entry[2], zero_balance_code=entry[3], @@ -192,8 +192,8 @@ class Command(BaseCommand): for index, entry in counterparty_data.iterrows(): counterparty = Counterparty( id=i, - loan_id=loan_dict[entry[0]], # loan FK reference - counterparty_id=entry[0], # counterparty ID is identical to loan ID + loan_identifier=loan_dict[entry[0]], # loan FK reference + counterparty_identifier=entry[0], # counterparty ID is identical to loan ID number_of_borrowers=entry[1], debt_to_income=entry[2], borrower_credit_score_at_origination=entry[3], @@ -222,7 +222,7 @@ class Command(BaseCommand): for index, entry in counterparty_state_data.iterrows(): counterparty_state = CounterpartyState( id=i, - counterparty_id=counterparty_dict[entry[0]], + counterparty_identifier=counterparty_dict[entry[0]], portfolio_snapshot_id=portfolio_snapshot_dict[entry[1]], borrower_credit_score_current=entry[2], coborrower_credit_score_current=entry[3], @@ -248,7 +248,7 @@ class Command(BaseCommand): for index, entry in property_collateral_data.iterrows(): property_collateral = PropertyCollateral( id=i, - loan_id=loan_dict[entry[0]], # loan FK reference + loan_identifier=loan_dict[entry[0]], # loan FK reference property_type=entry[1], number_of_units=entry[2], occupancy_status=entry[3], diff --git a/sflp_portfolio/management/commands/load_full_sflp_csv.py b/sflp_portfolio/management/commands/load_full_sflp_csv.py index 7f22bc1..62a343e 100644 --- a/sflp_portfolio/management/commands/load_full_sflp_csv.py +++ b/sflp_portfolio/management/commands/load_full_sflp_csv.py @@ -23,18 +23,26 @@ from django.core.management.base import BaseCommand from sflp_portfolio.models.counterparty import Counterparty +from sflp_portfolio.models.counterparty_state import CounterpartyState from sflp_portfolio.models.enforcement import Enforcement from sflp_portfolio.models.forbearance import Forbearance from sflp_portfolio.models.loan import Loan +from sflp_portfolio.models.loan_state import LoanState from sflp_portfolio.models.models import Portfolio from sflp_portfolio.models.models import PortfolioSnapshot from sflp_portfolio.models.property_collateral import PropertyCollateral -from sflp_portfolio.models.repayment_schedule import RepaymentSchedule +from sflp_portfolio.models.property_collateral_state import PropertyCollateralState +column_datatypes = { + 'zero_balance_code': pd.Int64Dtype(), + 'loan_holdback_indicator': pd.Int64Dtype(), + 'counterparty_identifier': str +} class Command(BaseCommand): - help = 'Imports SFLP data' + help = 'Imports Segmented SFLP data (All Models)' + # Clean up the database Portfolio.objects.all().delete() PortfolioSnapshot.objects.all().delete() Loan.objects.all().delete() @@ -42,236 +50,322 @@ class Command(BaseCommand): Enforcement.objects.all().delete() Forbearance.objects.all().delete() PropertyCollateral.objects.all().delete() - RepaymentSchedule.objects.all().delete() - data = pd.read_csv("sflp_portfolio/fixtures/test.csv", sep='|', index_col=None, low_memory=False, na_values=None, - true_values=['Y'], false_values=['N']) + # + # Portfolio data + # + portfolio_data = pd.read_csv("./sflp_portfolio/fixtures/portfolio.csv", sep='|', index_col=None, low_memory=False, + na_values=None, + true_values=['Y'], false_values=['N']) + print('Loaded Portfolio Data') + + portfolio_dict = {} + for index, entry in portfolio_data.iterrows(): + portfolio, _ = Portfolio.objects.update_or_create( + name=entry[0], + description='Test SFLP Portfolio', + ) + portfolio.save() + portfolio_dict[entry[0]] = portfolio + + print('Created Portfolio Entities') + # + # Portfolio snapshot data + # + portfolio_snapshot_data = pd.read_csv("./sflp_portfolio/fixtures/portfolio_snapshot.csv", sep='|', index_col=None, + low_memory=False, na_values=None, + true_values=['Y'], false_values=['N']) - data.fillna(np.nan, inplace=True) - data = data.replace({np.nan: None}) + print('Loaded Portfolio Snapshots') + portfolio_snapshot_dict = {} + for index, entry in portfolio_snapshot_data.iterrows(): + portfolio_snapshot, _ = PortfolioSnapshot.objects.update_or_create( + monthly_reporting_period=entry[0], + ) + portfolio_snapshot.save() + portfolio_snapshot_dict[entry[0]] = portfolio_snapshot - counterparty_data = [] - enforcement_data = [] - loan_data = [] - forbearance_data = [] - portfolio_data = [] - portfoliosnapshot_data = [] - propertycollateral_data = [] - repaymentschedule_data = [] + print('Created Portfolio Snapshots') + # + # Static Loan Data + # + loan_data = pd.read_csv("./sflp_portfolio/fixtures/loan.csv", sep='|', index_col=None, low_memory=False, + na_values=None, + true_values=['Y'], false_values=['N']) + print('Loaded Static Loan Data') + loan_data_list = [] i = 0 - for index, entry in data.iterrows(): + loan_dict = {} + for index, entry in loan_data.iterrows(): + # print('Loan: ', i) + loan = Loan( + id=i, + loan_identifier=entry[0], + portfolio=portfolio_dict[entry[1]], + channel=entry[2], + original_interest_rate=entry[3], + original_upb=entry[4], + original_loan_term=entry[5], + origination_date=entry[6], + first_payment_date=entry[7], + original_loan_to_value_ratio=entry[8], + loan_purpose=entry[9], + amortization_type=entry[10], + relocation_mortgage_indicator=entry[11], + high_balance_loan_indicator=entry[12], + mortgage_insurance_percentage=entry[13], + mortgage_insurance_type=entry[14], + original_combined_loan_to_value_ratio=entry[15] + + ) + # loan.save() + loan_data_list.append(loan) + loan_dict[entry[0]] = loan i += 1 - if (i > 4): - break - print(i) - if entry[43] == 1.0: - entry[43] = '01' - if entry[101] == 7.0: - entry[101] = '7' + Loan.objects.bulk_create(loan_data_list) + print('Created Static Loan Entries') + # + # Dynamic Loan Data + # - if entry[78] is None: - entry[78] = 'N' - portfolio, _ = Portfolio.objects.update_or_create( - name='test.csv', - description='Test Portfolio', - reference_pool_id=entry[0], - deal_name=entry[103], - ) - portfolio.save() + loan_state_data = pd.read_csv("./sflp_portfolio/fixtures/loan_state.csv", sep='|', index_col=None, na_values=None, + true_values=['Y'], false_values=['N'], dtype=column_datatypes) - portfoliosnapshot, _ = PortfolioSnapshot.objects.update_or_create( - monthly_reporting_period=entry[2], - ) - portfoliosnapshot.save() + loan_state_data = loan_state_data.replace({np.nan: None}) - loan = Loan( - portfolio=portfolio, - snapshot=portfoliosnapshot, - loan_identifier=str(entry[1]), - channel=CHANNEL_DICT[entry[3]], - seller_name=entry[4], - servicer_name=entry[5], - master_servicer=entry[6], - original_interest_rate=entry[7], - current_interest_rate=entry[8], - original_upb=entry[9], - upb_at_issuance=entry[10], - current_actual_upb=entry[11], - original_loan_term=entry[12], - origination_date=pd.to_datetime(entry[13], format="%m%Y"), - first_payment_date=pd.to_datetime(entry[14], format="%m%Y"), - loan_age=entry[15], - maturity_date=pd.to_datetime(entry[18], format="%m%Y"), - original_loan_to_value_ratio=entry[19], - original_combined_loan_to_value_ratio=entry[20], - loan_purpose=LOAN_PURPOSE_DICT[entry[26]], - mortgage_insurance_percentage=entry[33], - amortization_type=AMORTIZATION_DICT[entry[34]], - prepayment_penalty_indicator=entry[35], - interest_only_loan_indicator=entry[36], - interest_only_first_principal_and_interest_payment_date= - pd.to_datetime(entry[37], format="%m%Y"), - current_loan_delinquency_status=entry[39], - mortgage_insurance_cancellation_indicator=entry[42], - mortgage_insurance_type=MORTGAGE_INSURANCE_DICT[entry[72]], - servicing_activity_indicator=entry[73], - current_period_credit_event_net_gain_or_loss=entry[76], - cumulative_credit_event_net_gain_or_loss=entry[77], - special_eligibility_program=ELIGIBILITY_DICT[entry[78]], - relocation_mortgage_indicator=entry[80], - loan_holdback_indicator=LOAN_HOLDBACK_DICT[entry[82]], - loan_holdback_effective_date=entry[83], - delinquent_accrued_interest=entry[84], - high_balance_loan_indicator=entry[86], - arm_initial_fixed_rate_period_less_than_5_yr=entry[87], - arm_product_type=entry[88], - initial_fixed_rate_period=entry[89], - interest_rate_adjustment_frequency=entry[90], - next_interest_rate_adjustment_date=entry[91], - next_payment_change_date=entry[92], - index=entry[93], - arm_cap_structure=entry[94], - initial_interest_rate_cap=entry[95], - periodic_interest_rate_cap=entry[96], - lifetime_interest_rate_cap=entry[97], - mortgage_margin=entry[98], - arm_balloon_indicator=entry[99], - arm_plan_number=entry[100], - high_loan_to_value_refinance_option_indicator=entry[102], - repurchase_make_whole_proceeds_flag=entry[104], + print("Loaded Dynamic Loan Data") + + loan_state_data_list = [] + i = 0 + for index, entry in loan_state_data.iterrows(): + # print('LoanState: ', i) + loan_state = LoanState( + id=i, + loan_identifier=loan_dict[entry[0]], + portfolio_snapshot_id=portfolio_snapshot_dict[entry[1]], + high_loan_to_value_refinance_option_indicator=entry[2], + zero_balance_code=entry[3], + zero_balance_effective_date=entry[4], + upb_at_the_time_of_removal=entry[5], + total_principal_current=entry[6], + last_paid_installment_date=entry[7], + months_to_amortization=entry[8], + mortgage_insurance_cancellation_indicator=entry[9], + scheduled_principal_current=entry[10], + unscheduled_principal_current=entry[11], + zero_balance_code_change_date=entry[12], + loan_holdback_indicator=entry[13], + loan_holdback_effective_date=entry[14], + next_interest_rate_adjustment_date=entry[15], + next_payment_change_date=entry[16], + servicer_name=entry[17], + current_interest_rate=entry[18], + current_actual_upb=entry[19], + loan_age=entry[20], + remaining_months_to_legal_maturity=entry[21], + remaining_months_to_maturity=entry[22], + maturity_date=entry[23], + servicing_activity_indicator=entry[24], + repayment_history=entry[25] ) - loan.save() + loan_state_data_list.append(loan_state) + i += 1 + LoanState.objects.bulk_create(loan_state_data_list) + print("Created Dynamic Loan Data") + + # + # Static Counterparty data + # + counterparty_data = pd.read_csv("./sflp_portfolio/fixtures/counterparty.csv", sep='|', index_col=None, + low_memory=False, na_values=None, + true_values=['Y'], false_values=['N'], dtype=column_datatypes) + + print("Loaded Static Counterparty Data") + + counterparty_data_list = [] + i = 0 + counterparty_dict = {} + for index, entry in counterparty_data.iterrows(): counterparty = Counterparty( - loan_id=loan, - number_of_borrowers=entry[21], - debt_to_income=entry[22], - borrower_credit_score_at_origination=entry[23], - coborrower_credit_score_at_origination=entry[24], - first_time_home_buyer_indicator=FIRST_TIME_DICT[entry[25]], - borrower_credit_score_at_issuance=entry[68], - coborrower_credit_score_at_issuance=entry[69], - borrower_credit_score_current=entry[70], - coborrower_credit_score_current=entry[71], + id=i, + loan_identifier=loan_dict[entry[0]], # loan FK reference + counterparty_identifier=entry[0], # counterparty ID is identical to loan ID + number_of_borrowers=entry[1], + debt_to_income=entry[2], + borrower_credit_score_at_origination=entry[3], + coborrower_credit_score_at_origination=entry[4], + first_time_home_buyer_indicator=entry[5], ) - # counterparty.save() + counterparty_data_list.append(counterparty) + counterparty_dict[entry[0]] = counterparty + i += 1 - enforcement = Enforcement( - loan_id=loan, - foreclosure_date=entry[51], - disposition_date=entry[52], - foreclosure_costs=entry[53], - asset_recovery_costs=entry[55], - net_sales_proceeds=entry[58], - credit_enhancement_proceeds=entry[59], - repurchase_make_whole_proceeds=entry[60], - other_foreclosure_proceeds=entry[61], - original_list_start_date=entry[64], - original_list_price=entry[65], - current_list_start_date=entry[66], - current_list_price=entry[67], - foreclosure_principal_writeoff_amount=entry[79], - ) - # enforcement.save() + Counterparty.objects.bulk_create(counterparty_data_list) + print("Created Static Counterparty Data") - forbearance = Forbearance( - loan_id=loan, - modification_flag=entry[41], - noninterest_bearing_upb=entry[62], - principal_forgiveness_amount=entry[63], - current_period_modification_loss_amount=entry[74], - cumulative_modification_loss_amount=entry[75], - borrower_assistance_plan=BORROWER_PLAN_DICT[entry[101]], - alternative_delinquency_resolution=DELINQUENCY_DICT[entry[105]], - alternative_delinquency_resolution_count=entry[106], - total_deferral_amount=entry[107], + # + # Counterparty State data + # + counterparty_state_data = pd.read_csv("./sflp_portfolio/fixtures/counterparty_state.csv", sep='|', index_col=None, + low_memory=False, na_values=None, + true_values=['Y'], false_values=['N'], dtype=column_datatypes) + counterparty_state_data = counterparty_state_data.replace({np.nan: None}) + + print("Loaded Counterparty State Data") + + counterparty_state_data_list = [] + i = 0 + for index, entry in counterparty_state_data.iterrows(): + counterparty_state = CounterpartyState( + id=i, + counterparty_identifier=counterparty_dict[entry[0]], + portfolio_snapshot_id=portfolio_snapshot_dict[entry[1]], + borrower_credit_score_current=entry[2], + coborrower_credit_score_current=entry[3], ) - # forbearance.save() - - propertycollateral = PropertyCollateral( - loan_id=loan, - property_type=PROPERTY_DICT[entry[27]], - number_of_units=entry[28], - occupancy_status=OCCUPANCY_DICT[entry[29]], - property_state=entry[30], - metropolitan_statistical_area=entry[31], - zip_code_short=entry[32], - property_preservation_and_repair_costs=entry[54], - miscellaneous_holding_expenses_and_credits=entry[56], - associated_taxes_for_holding_property=entry[57], - property_valuation_method=PROPERTY_VALUATION_DICT[entry[85]], + counterparty_state_data_list.append(counterparty_state) + i += 1 + + CounterpartyState.objects.bulk_create(counterparty_state_data_list) + print("Created Counterparty State Data") + + # + # Static Property collateral data + # + property_collateral_data = pd.read_csv("./sflp_portfolio/fixtures/property_collateral.csv", sep='|', index_col=None, + low_memory=False, na_values=None, + true_values=['Y'], false_values=['N'], dtype=column_datatypes) + + print("Loaded Static Collateral Data") + + property_collateral_data_list = [] + i = 0 + property_collateral_dict = {} + for index, entry in property_collateral_data.iterrows(): + property_collateral = PropertyCollateral( + id=i, + loan_identifier=loan_dict[entry[0]], # loan FK reference + property_type=entry[1], + number_of_units=entry[2], + occupancy_status=entry[3], + property_state=entry[4], + metropolitan_statistical_area=entry[5], + zip_code_short=entry[6] ) - # propertycollateral.save() - - repaymentschedule = RepaymentSchedule( - loan_id=loan, - remaining_months_to_legal_maturity=entry[16], - remaining_months_to_maturity=entry[17], - months_to_amortization=entry[38], - loan_payment_history=entry[40], - zero_balance_code=ZERO_BALANCE_DICT[entry[43]], - zero_balance_effective_date=entry[44], - upb_at_the_time_of_removal=entry[45], - repurchase_date=entry[46], - scheduled_principal_current=entry[47], - total_principal_current=entry[48], - unscheduled_principal_current=entry[49], - last_paid_installment_date=entry[50], - zero_balance_code_change_date=entry[81], + property_collateral_data_list.append(property_collateral) + i += 1 + property_collateral_dict[entry[0]] = property_collateral + PropertyCollateral.objects.bulk_create(property_collateral_data_list) + print("Created Property Collateral Static Data") + + # + # Property Collateral State data + # + property_collateral_state_data = pd.read_csv("./sflp_portfolio/fixtures/property_collateral_state.csv", sep='|', + index_col=None, + low_memory=False, na_values=None, + true_values=['Y'], false_values=['N'], dtype=column_datatypes) + property_collateral_state_data = property_collateral_state_data.replace({np.nan: None}) + print("Loaded Property Collateral State Data") + + property_collateral_state_data_list = [] + i = 0 + for index, entry in property_collateral_state_data.iterrows(): + property_collateral_state = PropertyCollateralState( + id=i, + property_collateral_id=property_collateral_dict[entry[0]], + portfolio_snapshot_id=portfolio_snapshot_dict[entry[1]], + property_preservation_and_repair_costs=entry[2], + miscellaneous_holding_expenses_and_credits=entry[3], + associated_taxes_for_holding_property=entry[4], + property_valuation_method=entry[5], ) - # repaymentschedule.save() - - # portfolio_data.append(portfolio) - # portfoliosnapshot_data.append(portfoliosnapshot) - # loan_data.append(loan) - - counterparty_data.append(counterparty) - enforcement_data.append(enforcement) - forbearance_data.append(forbearance) - propertycollateral_data.append(propertycollateral) - repaymentschedule_data.append(repaymentschedule) - - # Portfolio.objects.bulk_create(portfolio_data) - # PortfolioSnapshot.objects.bulk_create(portfoliosnapshot_data) - # Loan.objects.bulk_create(loan_data) - - Counterparty.objects.bulk_create(counterparty_data) - Enforcement.objects.bulk_create(enforcement_data) - Forbearance.objects.bulk_create(forbearance_data) - PropertyCollateral.objects.bulk_create(propertycollateral_data) - RepaymentSchedule.objects.bulk_create(repaymentschedule_data) - - # # Delete existing objects - # Portfolio.objects.all().delete() + property_collateral_state_data_list.append(property_collateral_state) + i += 1 + + PropertyCollateralState.objects.bulk_create(property_collateral_state_data_list) + print("Created Property Collateral State Data") + # - # # Import data from file - # data = pd.read_csv("co.csv", header='infer', delimiter=',') + # Forbearance data # - # indata = [] - # for index, entry in data.iterrows(): - # pr = Project.objects.get(pk=entry['PROJECT']) - # co = Contractor( - # contractor_identifier=entry['PK'], - # project=pr, - # is_sme=entry['SME'], - # contractor_legal_entity_identifier=entry['NATIONALID'], - # name_of_contractor=entry['OFFICIALNAME'], - # address=entry['ADDRESS'], - # town=entry['TOWN'], - # postal_code=entry['POSTAL_CODE'], - # country=entry['COUNTRY'], - # phone=entry['PHONE'], - # email=entry['E_MAIL'], - # fax=entry['FAX'], - # region=entry['NUTS'], - # website=entry['URL']) + forbearance_data = pd.read_csv("./sflp_portfolio/fixtures/forbearance.csv", sep='|', + index_col=None, + low_memory=False, na_values=None, + true_values=['Y'], false_values=['N'], dtype=column_datatypes) + forbearance_data = forbearance_data.replace({np.nan: None}) + print("Loaded Forbearance Data") + + forbearance_data_list = [] + i = 0 + for index, entry in forbearance_data.iterrows(): + forbearance = Forbearance( + id=i, + loan_identifier=loan_dict[entry[0]], + portfolio_snapshot_id=portfolio_snapshot_dict[entry[1]], + current_loan_delinquency_status=entry[2], + modification_flag=entry[3], + noninterest_bearing_upb=entry[4], + principal_forgiveness_amount=entry[5], + current_period_modification_loss_amount=entry[6], + cumulative_modification_loss_amount=entry[7], + current_period_credit_event_net_gain_or_loss=entry[8], + delinquent_accrued_interest=entry[11], + borrower_assistance_plan=entry[12], + alternative_delinquency_resolution=entry[13], + alternative_delinquency_resolution_count=entry[14], + total_deferral_amount=entry[15], + ) + forbearance_data_list.append(forbearance) + i += 1 + + Forbearance.objects.bulk_create(forbearance_data_list) + print("Created Forbearance Data") + # - # indata.append(co) + # Enforcement data # - # Contractor.objects.bulk_create(indata) + enforcement_data = pd.read_csv("./sflp_portfolio/fixtures/enforcement.csv", sep='|', + index_col=None, + low_memory=False, na_values=None, + true_values=['Y'], false_values=['N'], dtype=column_datatypes) + enforcement_data = enforcement_data.replace({np.nan: None}) + print("Loaded Enforcement Data") + + enforcement_data_list = [] + i = 0 + for index, entry in enforcement_data.iterrows(): + enforcement = Enforcement( + id=i, + loan_identifier=loan_dict[entry[0]], + property_collateral_identifier=property_collateral_dict[entry[0]], + portfolio_snapshot_id=portfolio_snapshot_dict[entry[1]], + repurchase_date=entry[2], + foreclosure_date=entry[3], + disposition_date=entry[4], + foreclosure_costs=entry[5], + asset_recovery_costs=entry[6], + net_sales_proceeds=entry[7], + credit_enhancement_proceeds=entry[8], + repurchase_make_whole_proceeds=entry[9], + other_foreclosure_proceeds=entry[10], + original_list_start_date=entry[11], + original_list_price=entry[12], + current_list_start_date=entry[13], + current_list_price=entry[14], + cumulative_credit_event_net_gain_or_loss=entry[15], + foreclosure_principal_writeoff_amount=entry[16], + repurchase_make_whole_proceeds_flag=entry[17], + ) + enforcement_data_list.append(enforcement) + i += 1 + + Enforcement.objects.bulk_create(enforcement_data_list) + print("Created Enforcement Data") def handle(self, *args, **options): - self.stdout.write(self.style.SUCCESS('Successfully inserted SFLP data into db')) + self.stdout.write(self.style.SUCCESS('Successfully inserted full SFLP data into db')) diff --git a/sflp_portfolio/management/commands/load_static_sflp_csv.py b/sflp_portfolio/management/commands/load_static_sflp_csv.py index fcb68c5..f384941 100644 --- a/sflp_portfolio/management/commands/load_static_sflp_csv.py +++ b/sflp_portfolio/management/commands/load_static_sflp_csv.py @@ -116,8 +116,8 @@ class Command(BaseCommand): if i < LOAN_COUNT: counterparty = Counterparty.objects.create( id=i, - loan_id=loan_dict[entry[0]], # loan FK reference - counterparty_id=entry[0], # counterparty ID identical to loan ID + loan_identifier=loan_dict[entry[0]], # loan FK reference + counterparty_identifier=entry[0], # counterparty ID identical to loan ID number_of_borrowers=entry[1], debt_to_income=entry[2], borrower_credit_score_at_origination=entry[3], @@ -137,7 +137,7 @@ class Command(BaseCommand): if i < LOAN_COUNT: property_collateral = PropertyCollateral.objects.create( id=i, - loan_id=loan_dict[entry[0]], # loan FK reference + loan_identifier=loan_dict[entry[0]], # loan FK reference property_type=entry[1], number_of_units=entry[2], occupancy_status=entry[3], diff --git a/sflp_portfolio/models/counterparty.py b/sflp_portfolio/models/counterparty.py index de67b50..f5e406e 100644 --- a/sflp_portfolio/models/counterparty.py +++ b/sflp_portfolio/models/counterparty.py @@ -39,7 +39,7 @@ class Counterparty(models.Model): # IDENTIFICATION FIELDS # - counterparty_id = models.TextField(blank=True, null=True, + counterparty_identifier = models.TextField(blank=True, null=True, help_text='A unique identifier for the counterparty.') """A unique identifier for the counterparty.""" @@ -48,7 +48,7 @@ class Counterparty(models.Model): # - loan_id = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, + loan_identifier = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, help_text='The loan ID to which the Counterparty links.Documentation') """The loan ID to which the Counterparty links.""" @@ -96,7 +96,7 @@ class Counterparty(models.Model): def __str__(self): """String representing the data object""" - return str(self.counterparty_id) + return "Counterparty of Loan " + str(self.loan_identifier) def get_absolute_url(self): """Absolute URL where the data point can be edited""" diff --git a/sflp_portfolio/models/counterparty_state.py b/sflp_portfolio/models/counterparty_state.py index 9a5319b..f371403 100644 --- a/sflp_portfolio/models/counterparty_state.py +++ b/sflp_portfolio/models/counterparty_state.py @@ -41,7 +41,7 @@ class CounterpartyState(models.Model): help_text="The portfolio snapshot ID to which the Counterparty State corresponds.") # counterparty ID Foreign Key - counterparty_id = models.ForeignKey(Counterparty, on_delete=models.CASCADE, blank=True, null=True, + counterparty_identifier = models.ForeignKey(Counterparty, on_delete=models.CASCADE, blank=True, null=True, help_text="The counterparty ID to which the Counterparty State corresponds.") @@ -68,7 +68,7 @@ class CounterpartyState(models.Model): def __str__(self): """String representing the data object.""" - return "State of " + str(self.counterparty_id) + return "State of Counterparty" + str(self.counterparty_identifier) def get_absolute_url(self): """Absolute URL where the data point can be edited""" diff --git a/sflp_portfolio/models/enforcement.py b/sflp_portfolio/models/enforcement.py index fe623be..a7949d3 100644 --- a/sflp_portfolio/models/enforcement.py +++ b/sflp_portfolio/models/enforcement.py @@ -22,6 +22,7 @@ from django.urls import reverse from sflp_portfolio.models.loan import Loan +from sflp_portfolio.models.models import PortfolioSnapshot from sflp_portfolio.models.property_collateral import PropertyCollateral @@ -44,10 +45,14 @@ class Enforcement(models.Model): blank=True) - loan_id = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, + loan_identifier = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, help_text='The loan ID to which the Enforcement activity links.Documentation') """The loan ID to which the Enforcement activity links.""" + portfolio_snapshot_id = models.ForeignKey(PortfolioSnapshot, on_delete=models.CASCADE, blank=True, null=True, + help_text="The portfolio snapshot ID to which the Enforcement belongs") + """The snapshot to which the enforcement corresponds""" + # # DYNAMIC DATA PROPERTIES # @@ -124,7 +129,7 @@ class Enforcement(models.Model): def __str__(self): """String representing the data object""" - return "Enforcement of " + str(self.property_collateral_identifier) + return "Enforcement of Loan " + str(self.loan_identifier) def get_absolute_url(self): diff --git a/sflp_portfolio/models/forbearance.py b/sflp_portfolio/models/forbearance.py index 1380893..777ae17 100644 --- a/sflp_portfolio/models/forbearance.py +++ b/sflp_portfolio/models/forbearance.py @@ -23,6 +23,7 @@ from sflp_portfolio.models.loan import Loan from sflp_portfolio.models.model_choices import * +from sflp_portfolio.models.models import PortfolioSnapshot class Forbearance(models.Model): @@ -41,21 +42,19 @@ class Forbearance(models.Model): # FOREIGN KEYS # - loan_id = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, + loan_identifier = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, help_text='The loan ID to which the Forbearance activity links.Documentation') """The loan ID to which the Forbearance activity links.""" + portfolio_snapshot_id = models.ForeignKey(PortfolioSnapshot, on_delete=models.CASCADE, blank=True, null=True, + help_text="The portfolio snapshot ID to which the Forbearance belongs") + """The snapshot to which the forbearance corresponds""" + # # DYNAMIC DATA PROPERTIES # - loan_holdback_effective_date = models.DateField(blank=True, null=True, - help_text='The date of the latest Loan Holdback indicator change.Documentation') - """""" - loan_holdback_indicator = models.IntegerField(blank=True, null=True, choices=LOAN_HOLDBACK_INDICATOR_CHOICES, - help_text='An indicator that denotes if a loan has been moved temporarily into a ‘hold’ status to allow Fannie Mae to further evaluate unique situations that may otherwise result in a credit event or loan removal. Such situations may include loans with reported data anomalies, loans currently in forbearance due to a natural disaster or loans refinanced under the High LTV program that will continue to be included in the reference pool.Documentation') - """""" current_period_credit_event_net_gain_or_loss = models.FloatField(blank=True, null=True, help_text='The net realized gain or loss amount calculated for a mortgage loan resulting from a credit event for the corresponding reporting period.Documentation') @@ -118,7 +117,7 @@ class Forbearance(models.Model): def __str__(self): """String representing the data object""" - return "Forbearance of " + str(self.loan_id) + return "Forbearance of Loan " + str(self.loan_identifier) def get_absolute_url(self): """Absolute URL where the data point can be edited""" diff --git a/sflp_portfolio/models/loan.py b/sflp_portfolio/models/loan.py index c39aafe..9f78e59 100644 --- a/sflp_portfolio/models/loan.py +++ b/sflp_portfolio/models/loan.py @@ -143,6 +143,11 @@ class Loan(models.Model): help_text='For an adjustable-rate mortgage loan, the maximum percentage points the interest rate can adjust upward at the initial interest rate change date. Documentation') """""" + special_eligibility_program = models.IntegerField(blank=True, null=True, + choices=SPECIAL_ELIGIBILITY_PROGRAM_CHOICES, + help_text='A mortgage program with expanded eligibility criteria designed to increase and maintain home ownership. Documentation') + """""" + # # BOOKKEEPING FIELDS # diff --git a/sflp_portfolio/models/loan_state.py b/sflp_portfolio/models/loan_state.py index 18281ec..b09677b 100644 --- a/sflp_portfolio/models/loan_state.py +++ b/sflp_portfolio/models/loan_state.py @@ -40,10 +40,10 @@ class LoanState(models.Model): # Portfolio Snapshot ID Foreign Key portfolio_snapshot_id = models.ForeignKey(PortfolioSnapshot, on_delete=models.CASCADE, blank=True, null=True, help_text="The portfolio snapshot ID to which the Loan State belongs") - """""" + """The snapshot to which the loan state corresponds""" # Loan ID Foreign Key - loan_id = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, + loan_identifier = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, help_text='The loan ID to which the loan state links.Documentation') """The loan ID to which the loan state links.""" @@ -51,8 +51,6 @@ class LoanState(models.Model): # DYNAMIC DATA PROPERTIES # - - current_actual_upb = models.FloatField(blank=True, null=True, help_text='The current actual outstanding unpaid principal balance of a mortgage loan, reflective of payments actually received from the related borrower.Documentation') """""" @@ -61,10 +59,6 @@ class LoanState(models.Model): help_text='The rate of interest in effect for the periodic installment due.Documentation') """""" - - - - high_loan_to_value_refinance_option_indicator = models.BooleanField(blank=True, null=True, help_text='An indicator that denotes if an eligible original reference loan is refinanced under Fannie Mae’s HLTV refinance option, which results in such mortgage loan remaining in the Reference Pool, as further defined in each individual CRT document, if applicable.Documentation') """""" @@ -73,7 +67,6 @@ class LoanState(models.Model): help_text='For adjustable-rate loans, the description of the index on which adjustments to the interest rate are based.Documentation') """""" - interest_only_first_principal_and_interest_payment_date = models.DateField(blank=True, null=True, help_text='For interest-only loans, the month and year that the first monthly scheduled fully amortizing principal and interest payment is due.Documentation') """""" @@ -94,8 +87,6 @@ class LoanState(models.Model): help_text='The number of calendar months since the mortgage loans origination date. For purposes of calculating this data element, origination means the date on which the first full month of interest begins to accrue.Documentation') """""" - - master_servicer = models.TextField(blank=True, null=True, help_text='Fannie Mae.Documentation') """""" @@ -128,7 +119,6 @@ class LoanState(models.Model): help_text='An indicator that denotes whether the borrower is subject to a penalty for early payment of principal.Documentation') """""" - """""" servicer_name = models.TextField(blank=True, null=True, @@ -139,11 +129,6 @@ class LoanState(models.Model): help_text='An indicator that denotes a change in servicing activity during the corresponding reporting period.Documentation') """""" - special_eligibility_program = models.IntegerField(blank=True, null=True, - choices=SPECIAL_ELIGIBILITY_PROGRAM_CHOICES, - help_text='A mortgage program with expanded eligibility criteria designed to increase and maintain home ownership. Documentation') - """""" - upb_at_issuance = models.FloatField(blank=True, null=True, help_text='The unpaid principal balance of the loan as of the cut-off date of the reference pool.Documentation') """""" @@ -192,6 +177,18 @@ class LoanState(models.Model): help_text='The due date of the last paid installment that was collected for the mortgage loan.Documentation') """""" + loan_holdback_effective_date = models.DateField(blank=True, null=True, + help_text='The date of the latest Loan Holdback indicator change.Documentation') + """""" + + loan_holdback_indicator = models.IntegerField(blank=True, null=True, choices=LOAN_HOLDBACK_INDICATOR_CHOICES, + help_text='An indicator that denotes if a loan has been moved temporarily into a ‘hold’ status to allow Fannie Mae to further evaluate unique situations that may otherwise result in a credit event or loan removal. Such situations may include loans with reported data anomalies, loans currently in forbearance due to a natural disaster or loans refinanced under the High LTV program that will continue to be included in the reference pool.Documentation') + """""" + + repayment_history = models.TextField(blank=True, null=True, + help_text='The coded string of values that describes the payment performance of the loan over the most recent 24 months. The most recent month is located to the right..Documentation') + """The coded string of values that describes the payment performance of the loan over the most recent 24 months""" + # # BOOKKEEPING FIELDS # @@ -204,7 +201,7 @@ class LoanState(models.Model): def __str__(self): """String representing the data object""" - return "State of " + str(self.loan_id) + return "State of Loan " + str(self.loan_identifier) def get_absolute_url(self): """Absolute URL where the data point can be edited""" diff --git a/sflp_portfolio/models/property_collateral.py b/sflp_portfolio/models/property_collateral.py index 5a0bafe..31a912e 100644 --- a/sflp_portfolio/models/property_collateral.py +++ b/sflp_portfolio/models/property_collateral.py @@ -40,7 +40,7 @@ class PropertyCollateral(models.Model): # FOREIGN KEYS # - loan_id = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, + loan_identifier = models.ForeignKey(Loan, on_delete=models.CASCADE, blank=True, null=True, help_text='The loan ID to which the property collateral links.Documentation') """The loan ID to which the property collateral links.""" @@ -80,7 +80,7 @@ class PropertyCollateral(models.Model): def __str__(self): """String representing the data object""" - return "Collateral of " + str(self.loan_id) + return "Collateral of Loan " + str(self.loan_identifier) def get_absolute_url(self): """Absolute URL where the data point can be edited""" diff --git a/sflp_portfolio/models/property_collateral_state.py b/sflp_portfolio/models/property_collateral_state.py index 74ff8b5..639817b 100644 --- a/sflp_portfolio/models/property_collateral_state.py +++ b/sflp_portfolio/models/property_collateral_state.py @@ -77,7 +77,7 @@ class PropertyCollateralState(models.Model): def __str__(self): """String representing the data object""" - return "State of " + str(self.property_collateral_id) + return "State of Collateral " + str(self.property_collateral_id) def get_absolute_url(self):