diff --git a/Makefile b/Makefile
index 57ededb..e51d9f4 100644
--- a/Makefile
+++ b/Makefile
@@ -1,15 +1,21 @@
+.PRECIOUS : _data/raw/%.csv
+
+import/% : s3/$(AWS_STORAGE_BUCKET_NAME)/%.gz
+ python manage.py import_api_data --transaction-type $(word 1, $(subst _, , $*)) \
+ --year $(word 2, $(subst _, , $*))
+
+import/local/% : _data/raw/%.csv
+ python manage.py import_api_data --transaction-type $(word 1, $(subst _, , $*)) \
+ --year $(word 2, $(subst _, , $*)) \
+ --file $<
+
s3/$(AWS_STORAGE_BUCKET_NAME)/%.gz : %.gz
aws s3 cp $< s3://$$AWS_STORAGE_BUCKET_NAME
%.gz : _data/raw/%.csv
gzip -c $< > $@
-_data/raw/CON_%.csv :
- wget --no-use-server-timestamps \
- "https://login.cfis.sos.state.nm.us/api/DataDownload/GetCSVDownloadReport?year=$*&transactionType=CON&reportFormat=csv&fileName=$(notdir $@)" \
- -O $@
-
-data/raw/EXP_%.csv :
+_data/raw/%.csv :
wget --no-use-server-timestamps \
- "https://login.cfis.sos.state.nm.us/api/DataDownload/GetCSVDownloadReport?year=$*&transactionType=EXP&reportFormat=csv&fileName=$(notdir $@)" \
+ "https://login.cfis.sos.state.nm.us/api/DataDownload/GetCSVDownloadReport?year=$(word 2, $(subst _, , $*))&transactionType=$(word 1, $(subst _, , $*))&reportFormat=csv&fileName=$(notdir $@)" \
-O $@
diff --git a/camp_fin/management/commands/import_api_data.py b/camp_fin/management/commands/import_api_data.py
new file mode 100644
index 0000000..0b67a99
--- /dev/null
+++ b/camp_fin/management/commands/import_api_data.py
@@ -0,0 +1,692 @@
+import csv
+from datetime import datetime
+import gzip
+from itertools import groupby
+import os
+import re
+
+from dateutil.parser import parse, ParserError
+from django.core.exceptions import MultipleObjectsReturned
+from django.core.management import call_command
+from django.core.management.base import BaseCommand
+from django.db.models import Max, Sum
+from django.utils.text import slugify
+
+import boto3
+import requests
+from tqdm import tqdm
+
+from camp_fin import models
+
+
+class Command(BaseCommand):
+ help = """
+ Import data from the New Mexico Campaign Finance System:
+ https://login.cfis.sos.state.nm.us/#/dataDownload
+
+ Data will be retrieved from S3 unless a local CSV is specified as --file
+ """
+
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+
+ try:
+ self._next_entity_id = (
+ models.Entity.objects.aggregate(max_id=Max("user_id"))["max_id"] + 1
+ )
+ except TypeError:
+ self._next_entity_id = 1
+
+ self._cache = {
+ "state": {obj.postal_code: obj for obj in models.State.objects.iterator()},
+ "contact_type": {
+ obj.description: obj for obj in models.ContactType.objects.iterator()
+ },
+ "entity_type": {
+ obj.description: obj for obj in models.EntityType.objects.iterator()
+ },
+ "filing_type": {
+ obj.description: obj for obj in models.FilingType.objects.iterator()
+ },
+ "transaction_type": {
+ obj.description: obj
+ for obj in models.TransactionType.objects.iterator()
+ },
+ "loan_transaction_type": {
+ obj.description: obj
+ for obj in models.LoanTransactionType.objects.iterator()
+ },
+ "entity": {},
+ "candidate": {},
+ "campaign": {},
+ "pac": {},
+ "election_season": {},
+ "filing_period": {},
+ "filing": {},
+ "address": {},
+ "office": {},
+ "party": {},
+ }
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ "--transaction-type",
+ dest="transaction_type",
+ default="CON",
+ help="Type of transaction to import: CON, EXP (Default: CON)",
+ )
+ parser.add_argument(
+ "--year",
+ dest="year",
+ default="2023",
+ help="Year to import (Default: 2023)",
+ )
+ parser.add_argument(
+ "--file",
+ dest="file",
+ help="Absolute path of CSV file to import",
+ required=False,
+ )
+
+ def handle(self, *args, **options):
+ if options["transaction_type"] not in ("EXP", "CON"):
+ raise ValueError("Transaction type must be one of: EXP, CON")
+
+ if options["file"]:
+ f = open(options["file"], "r")
+
+ else:
+ s3 = boto3.client("s3")
+
+ resource_name = f"{options['transaction_type']}_{options['year']}.gz"
+
+ with open(resource_name, "wb") as download_location:
+ s3.download_fileobj(
+ os.getenv("AWS_STORAGE_BUCKET_NAME", "openness-project-nmid"),
+ resource_name,
+ download_location,
+ )
+
+ f = gzip.open(resource_name, "rt")
+
+ try:
+ if options["transaction_type"] == "CON":
+ self.import_contributions(f)
+
+ elif options["transaction_type"] == "EXP":
+ self.import_expenditures(f)
+
+ finally:
+ f.close()
+
+ self.total_filings(options["year"])
+ call_command("import_data", "--add-aggregates")
+
+ def fetch_from_cache(self, cache_entity, cache_key, model, model_kwargs):
+ try:
+ return self._cache[cache_entity][cache_key]
+ except KeyError:
+ deidentified_model_kwargs = {
+ k: v for k, v in model_kwargs.items() if k not in ("entity", "slug")
+ }
+
+ try:
+ obj = model.objects.get(**deidentified_model_kwargs)
+ except model.DoesNotExist:
+ obj = model.objects.create(**model_kwargs)
+ except model.MultipleObjectsReturned:
+ obj = model.objects.filter(**deidentified_model_kwargs).first()
+
+ self._cache[cache_entity][cache_key] = obj
+ return obj
+
+ def parse_date(self, date_str):
+ try:
+ return parse(date_str).date()
+ except ParserError:
+ self.stderr.write(
+ self.style.ERROR(f"Could not parse date from string '{date_str}'")
+ )
+ return None
+
+ def import_contributions(self, f):
+ reader = csv.DictReader(f)
+
+ key_func = lambda record: (record["OrgID"], record["Report Name"])
+ sorted_records = sorted(reader, key=key_func)
+
+ loans, special_events, transactions = [], [], []
+
+ for filing_group, records in groupby(tqdm(sorted_records), key=key_func):
+ for i, record in enumerate(records):
+ if i == 0:
+ try:
+ filing = self.make_filing(record)
+ except ValueError:
+ continue
+
+ models.LoanTransaction.objects.filter(filing=filing).delete()
+ models.SpecialEvent.objects.filter(filing=filing).delete()
+ models.Transaction.objects.filter(filing=filing).exclude(
+ transaction_type__description="Monetary Expenditure"
+ ).delete()
+
+ contributor = self.make_contributor(record)
+
+ if record["Contribution Type"] == "Loans Received":
+ loans.append(self.make_contribution(record, contributor, filing))
+
+ elif record["Contribution Type"] == "Special Event":
+ special_events.append(
+ self.make_contribution(record, contributor, filing)
+ )
+
+ elif "Contribution" in record["Contribution Type"]:
+ transactions.append(
+ self.make_contribution(record, contributor, filing)
+ )
+
+ else:
+ self.stderr.write(
+ f"Could not determine contribution type from record: {record['Contribution Type']}"
+ )
+
+ if len(transactions) >= 2500:
+ models.Transaction.objects.bulk_create(transactions)
+ transactions = []
+
+ models.LoanTransaction.objects.bulk_create(loans)
+ models.SpecialEvent.objects.bulk_create(special_events)
+ models.Transaction.objects.bulk_create(transactions)
+
+ def import_expenditures(self, f):
+ reader = csv.DictReader(f)
+
+ key_func = lambda record: (record["OrgID"], record["Report Name"])
+ sorted_records = sorted(reader, key=key_func)
+
+ expenditures = []
+
+ for filing_group, records in groupby(tqdm(sorted_records), key=key_func):
+ for i, record in enumerate(records):
+ if i == 0:
+ try:
+ filing = self.make_filing(record)
+ except ValueError:
+ continue
+
+ models.Transaction.objects.filter(
+ filing=filing,
+ transaction_type__description="Monetary Expenditure",
+ ).delete()
+
+ expenditures.append(self.make_contribution(record, None, filing))
+
+ if len(expenditures) >= 2500:
+ models.Transaction.objects.bulk_create(expenditures)
+ expenditures = []
+
+ models.Transaction.objects.bulk_create(expenditures)
+
+ def make_contributor(self, record):
+ state = self.fetch_from_cache(
+ "state",
+ record["Contributor State"],
+ models.State,
+ {"postal_code": record["Contributor State"]},
+ )
+
+ address = self.fetch_from_cache(
+ "address",
+ (
+ f"{record['Contributor Address Line 1']}{' ' + record['Contributor Address Line 2'] if record['Contributor Address Line 2'] else ''}",
+ record["Contributor City"],
+ state,
+ record["Contributor Zip Code"],
+ ),
+ models.Address,
+ dict(
+ street=f"{record['Contributor Address Line 1']}{' ' + record['Contributor Address Line 2'] if record['Contributor Address Line 2'] else ''}",
+ city=record["Contributor City"],
+ state=state,
+ zipcode=record["Contributor Zip Code"],
+ ),
+ )
+
+ contact_type = self.fetch_from_cache(
+ "contact_type",
+ record["Contributor Code"],
+ models.ContactType,
+ {"description": record["Contributor Code"]},
+ )
+
+ full_name = re.sub(
+ r"\s{2,}",
+ " ",
+ " ".join(
+ [
+ record["Prefix"],
+ record["First Name"],
+ record["Middle Name"],
+ record["Last Name"],
+ record["Suffix"],
+ ]
+ ),
+ ).strip()
+
+ contact_kwargs = {
+ "prefix": record["Prefix"],
+ "first_name": record["First Name"],
+ "middle_name": record["Middle Name"],
+ "last_name": record["Last Name"],
+ "suffix": record["Suffix"],
+ "occupation": record["Contributor Occupation"],
+ "company_name": record["Contributor Employer"],
+ "full_name": full_name,
+ }
+
+ try:
+ contact = models.Contact.objects.get(
+ **contact_kwargs,
+ status_id=0,
+ address=address,
+ contact_type=contact_type,
+ )
+ except models.Contact.DoesNotExist:
+ entity_type = self.fetch_from_cache(
+ "entity_type",
+ record["Contributor Code"][:24],
+ models.EntityType,
+ {"description": record["Contributor Code"][:24]},
+ )
+
+ entity = models.Entity.objects.create(
+ user_id=self._next_entity_id,
+ entity_type=entity_type,
+ )
+ contact = models.Contact.objects.create(
+ **contact_kwargs,
+ status_id=0,
+ address=address,
+ contact_type=contact_type,
+ entity=entity,
+ )
+
+ self._next_entity_id += 1
+
+ return contact
+
+ def make_pac(self, record, entity_type=None):
+ entity_type = self.fetch_from_cache(
+ "entity_type",
+ entity_type or record["Report Entity Type"],
+ models.EntityType,
+ {"description": entity_type or record["Report Entity Type"]},
+ )
+
+ entity = self.fetch_from_cache(
+ "entity",
+ (record["OrgID"], entity_type.description),
+ models.Entity,
+ {"user_id": record["OrgID"], "entity_type": entity_type},
+ )
+
+ return self.fetch_from_cache(
+ "pac",
+ record["Committee Name"],
+ models.PAC,
+ dict(
+ name=record["Committee Name"],
+ entity=entity,
+ slug=slugify(record["Committee Name"]),
+ ),
+ )
+
+ def make_filing(self, record):
+ if record["Report Entity Type"] == "Candidate":
+ # Create PAC associated with candidate
+ self.make_pac(record, entity_type="Political Committee")
+
+ entity_type = self.fetch_from_cache(
+ "entity_type",
+ "Candidate",
+ models.EntityType,
+ {"description": "Candidate"},
+ )
+
+ entity = self.fetch_from_cache(
+ "entity",
+ (record["OrgID"], "Candidate"),
+ models.Entity,
+ {"user_id": record["OrgID"], "entity_type": entity_type},
+ )
+
+ if any(
+ [
+ record["Candidate First Name"],
+ record["Candidate Last Name"],
+ ]
+ ):
+ full_name = re.sub(
+ r"\s{2,}",
+ " ",
+ " ".join(
+ [
+ record["Candidate First Name"],
+ record["Candidate Middle Name"],
+ record["Candidate Last Name"],
+ record["Candidate Suffix"],
+ ]
+ ),
+ ).strip()
+
+ candidate = self.fetch_from_cache(
+ "candidate",
+ full_name,
+ models.Candidate,
+ dict(
+ first_name=record["Candidate First Name"] or None,
+ middle_name=record["Candidate Middle Name"] or None,
+ last_name=record["Candidate Last Name"] or None,
+ suffix=record["Candidate Suffix"] or None,
+ full_name=full_name,
+ entity=entity,
+ slug=slugify(
+ " ".join(
+ [
+ record["Candidate First Name"],
+ record["Candidate Last Name"],
+ ]
+ )
+ ),
+ ),
+ )
+
+ # If an existing candidate was found, grab its entity.
+ if candidate.entity.user_id != record["OrgID"]:
+ entity = candidate.entity
+
+ else:
+ # Sometimes, a record says it is associated with a candidate,
+ # but a candidate name is not provided. This block attempts to
+ # look up the candidate based on committee name. If we cannot
+ # identify one candidate, we skip the record.
+ try:
+ candidate = (
+ models.Candidate.objects.filter(
+ campaign__committee_name=record["Committee Name"]
+ )
+ .distinct()
+ .get()
+ )
+ except models.Candidate.DoesNotExist:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Could not find candidate associated with committee {record['Committee Name']}. Skipping..."
+ )
+ )
+ raise ValueError
+ except models.Candidate.MultipleObjectsReturned:
+ self.stdout.write(
+ self.style.ERROR(
+ f"Found more than one candidate associated with committee {record['Committee Name']}. Skipping..."
+ )
+ )
+ raise ValueError
+
+ # This is fudged and should be drawn instead from a canonical list of offices and races.
+ # We need a campaign for committee finances to be associated with the candidate.
+ election_year = self.parse_date(record["Start of Period"]).year
+
+ election_season = self.fetch_from_cache(
+ "election_season",
+ (election_year, False, 0),
+ models.ElectionSeason,
+ dict(
+ year=self.parse_date(record["Start of Period"]).year,
+ special=False,
+ status_id=0,
+ ),
+ )
+
+ office = self.fetch_from_cache(
+ "office",
+ None,
+ models.Office,
+ dict(description="Not specified", status_id=0),
+ )
+
+ party = self.fetch_from_cache(
+ "party",
+ None,
+ models.PoliticalParty,
+ dict(name="Not specified"),
+ )
+
+ campaign = self.fetch_from_cache(
+ "campaign",
+ (
+ record["Committee Name"],
+ candidate.full_name,
+ election_season.year,
+ ),
+ models.Campaign,
+ dict(
+ committee_name=record["Committee Name"],
+ candidate=candidate,
+ election_season=election_season,
+ office=office,
+ political_party=party,
+ ),
+ )
+
+ filing_kwargs = {"entity": entity, "campaign": campaign}
+
+ elif record["Report Entity Type"]:
+ pac = self.make_pac(record)
+
+ filing_kwargs = {"entity": pac.entity}
+
+ else:
+ self.stderr.write(
+ self.style.ERROR(
+ f"Report entity type not provided. Skipping record {record}..."
+ )
+ )
+ raise ValueError
+
+ filing_type = self.fetch_from_cache(
+ "filing_type",
+ record["Report Name"][:24],
+ models.FilingType,
+ {"description": record["Report Name"][:24]},
+ )
+
+ filing_period = self.fetch_from_cache(
+ "filing_period",
+ (
+ record["Report Name"],
+ self.parse_date(record["Filed Date"]),
+ self.parse_date(record["Start of Period"]),
+ self.parse_date(record["End of Period"]),
+ ),
+ models.FilingPeriod,
+ dict(
+ description=record["Report Name"],
+ filing_date=(
+ self.parse_date(record["Filed Date"])
+ or self.parse_date(record["End of Period"])
+ ),
+ initial_date=self.parse_date(record["Start of Period"]),
+ due_date=self.parse_date(record["End of Period"]),
+ allow_no_activity=False,
+ exclude_from_cascading=False,
+ email_sent_status=0,
+ filing_period_type=filing_type,
+ ),
+ )
+
+ filing = self.fetch_from_cache(
+ "filing",
+ (
+ filing_kwargs["entity"].user_id,
+ filing_period.id,
+ self.parse_date(record["End of Period"]),
+ ),
+ models.Filing,
+ dict(
+ filing_period=filing_period,
+ date_closed=self.parse_date(record["End of Period"]),
+ final=True,
+ **filing_kwargs,
+ ),
+ )
+
+ return filing
+
+ def make_contribution(self, record, contributor, filing):
+ if contributor:
+ address_kwargs = dict(
+ address=f"{record['Contributor Address Line 1']}{' ' + record['Contributor Address Line 2'] if record['Contributor Address Line 2'] else ''}",
+ city=record["Contributor City"],
+ state=record["Contributor State"],
+ zipcode=record["Contributor Zip Code"],
+ )
+
+ if record["Contribution Type"] == "Loans Received":
+ transaction_type = self.fetch_from_cache(
+ "loan_transaction_type",
+ "Payment",
+ models.LoanTransactionType,
+ {"description": "Payment"},
+ )
+
+ loan, _ = models.Loan.objects.get_or_create(
+ amount=record["Transaction Amount"],
+ received_date=self.parse_date(record["Transaction Date"]),
+ check_number=record["Check Number"],
+ status_id=0,
+ contact=contributor,
+ company_name=contributor.company_name or "Not specified",
+ filing=filing,
+ **address_kwargs,
+ )
+
+ contribution = models.LoanTransaction(
+ amount=record["Transaction Amount"],
+ transaction_date=self.parse_date(record["Transaction Date"]),
+ transaction_status_id=0,
+ loan=loan,
+ filing=filing,
+ transaction_type=transaction_type,
+ )
+
+ elif record["Contribution Type"] == "Special Event":
+ address_kwargs.pop("state")
+
+ contribution = models.SpecialEvent(
+ anonymous_contributions=record["Transaction Amount"],
+ event_date=self.parse_date(record["Transaction Date"]),
+ admission_price=0,
+ attendance=0,
+ total_admissions=0,
+ total_expenditures=0,
+ transaction_status_id=0,
+ sponsors=contributor.company_name or "Not specified",
+ filing=filing,
+ **address_kwargs,
+ )
+
+ elif "Contribution" in record["Contribution Type"]:
+ transaction_type = self.fetch_from_cache(
+ "transaction_type",
+ "Monetary contribution",
+ models.TransactionType,
+ {
+ "description": "Monetary contribution",
+ "contribution": True,
+ "anonymous": False,
+ },
+ )
+
+ contribution = models.Transaction(
+ amount=record["Transaction Amount"],
+ received_date=self.parse_date(record["Transaction Date"]),
+ check_number=record["Check Number"],
+ description=record["Description"][:74],
+ contact=contributor,
+ full_name=contributor.full_name,
+ filing=filing,
+ transaction_type=transaction_type,
+ **address_kwargs,
+ company_name=contributor.full_name,
+ occupation=record["Contributor Occupation"],
+ )
+
+ else:
+ address_kwargs = dict(
+ address=f"{record['Payee Address 1']}{' ' + record['Payee Address 2'] if record['Payee Address 2'] else ''}",
+ city=record["Payee City"],
+ state=record["Payee State"],
+ zipcode=record["Payee Zip Code"],
+ )
+
+ transaction_type = self.fetch_from_cache(
+ "transaction_type",
+ "Monetary Expenditure",
+ models.TransactionType,
+ {
+ "description": "Monetary Expenditure",
+ "contribution": False,
+ "anonymous": False,
+ },
+ )
+
+ payee_full_name = re.sub(
+ r"\s{2,}",
+ " ",
+ " ".join(
+ [
+ record["Payee Prefix"],
+ record["Payee First Name"],
+ record["Payee Middle Name"],
+ record["Payee Last Name"],
+ record["Payee Suffix"],
+ ]
+ ),
+ ).strip()
+
+ contribution = models.Transaction(
+ amount=record["Expenditure Amount"],
+ received_date=self.parse_date(record["Expenditure Date"]),
+ description=(record["Description"] or record["Expenditure Type"])[:74],
+ full_name=payee_full_name,
+ company_name=payee_full_name,
+ filing=filing,
+ transaction_type=transaction_type,
+ **address_kwargs,
+ )
+
+ return contribution
+
+ def total_filings(self, year):
+ for filing in models.Filing.objects.filter(
+ filing_period__filing_date__year=year
+ ).iterator():
+ contributions = filing.contributions().aggregate(total=Sum("amount"))
+ expenditures = filing.expenditures().aggregate(total=Sum("amount"))
+ loans = filing.loans().aggregate(total=Sum("amount"))
+
+ filing.total_contributions = contributions["total"] or 0
+ filing.total_expenditures = expenditures["total"] or 0
+ filing.total_loans = loans["total"] or 0
+
+ filing.closing_balance = filing.opening_balance or 0 + (
+ filing.total_contributions
+ + filing.total_loans
+ - filing.total_expenditures
+ )
+
+ filing.save()
+
+ self.stdout.write(f"Totalled {filing}")
diff --git a/camp_fin/management/commands/import_data.py b/camp_fin/management/commands/import_data.py
index d8888f8..90c6e93 100644
--- a/camp_fin/management/commands/import_data.py
+++ b/camp_fin/management/commands/import_data.py
@@ -407,37 +407,6 @@ def loadLoanTransactions(self):
else:
self.doETL("loantransaction")
- def makeAllExpenditureView(self):
- self.loadLoanTransactions()
-
- view = """
- SELECT
- transaction.filing_id,
- transaction.id,
- transaction.amount,
- transaction.full_name,
- transaction_type.description,
- transaction_type.contribution
- FROM camp_fin_transaction AS transaction
- JOIN camp_fin_transactiontype AS transaction_type
- ON transaction.transaction_type_id = transaction_type.id
- WHERE transaction_type.contribution = FALSE
- UNION
- SELECT
- loan_transaction.filing_id,
- loan_transaction.id,
- loan_transaction.amount,
- loan.full_name,
- loan_transaction_type.description,
- FALSE as contribution
- FROM camp_fin_loantransaction AS loan_transaction
- JOIN camp_fin_loantransactiontype AS loan_transaction_type
- ON loan_transaction.transaction_type_id = loan_transaction_type.id
- JOIN camp_fin_loan AS loan
- ON loan_transaction.loan_id = loan.id
- WHERE loan_transaction_type.description = 'Payment'
- """
-
def makeLoanBalanceView(self, aggregates_only=False):
if not aggregates_only:
self.loadLoanTransactions()
diff --git a/camp_fin/management/commands/import_data_2023.py b/camp_fin/management/commands/import_data_2023.py
deleted file mode 100644
index 7788882..0000000
--- a/camp_fin/management/commands/import_data_2023.py
+++ /dev/null
@@ -1,500 +0,0 @@
-import csv
-from datetime import datetime
-import gzip
-from itertools import groupby
-import os
-import re
-
-from dateutil.parser import parse
-from django.core.exceptions import MultipleObjectsReturned
-from django.core.management import call_command
-from django.core.management.base import BaseCommand
-from django.db.models import Max, Sum
-from django.utils.text import slugify
-
-import boto3
-import requests
-from tqdm import tqdm
-
-from camp_fin import models
-
-
-class Command(BaseCommand):
- help = "https://docs.google.com/spreadsheets/d/1bKF74KRMXiUaWttamG0lHHh2yLTSE7ctpkm6i8wSwaM/edit?usp=sharing"
-
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
-
- try:
- self._next_entity_id = (
- models.Entity.objects.aggregate(max_id=Max("user_id"))["max_id"] + 1
- )
- except TypeError:
- self._next_entity_id = 1
-
- self._cache = {
- "state": {obj.postal_code: obj for obj in models.State.objects.iterator()},
- "contact_type": {
- obj.description: obj for obj in models.ContactType.objects.iterator()
- },
- "entity_type": {
- obj.description: obj for obj in models.EntityType.objects.iterator()
- },
- "filing_type": {
- obj.description: obj for obj in models.FilingType.objects.iterator()
- },
- "transaction_type": {
- obj.description: obj
- for obj in models.TransactionType.objects.iterator()
- },
- "loan_transaction_type": {
- obj.description: obj
- for obj in models.LoanTransactionType.objects.iterator()
- },
- "entity": {},
- "candidate": {},
- "campaign": {},
- "pac": {},
- "election_season": {},
- "filing_period": {},
- "filing": {},
- "address": {},
- }
-
- def add_arguments(self, parser):
- parser.add_argument(
- "--transaction-type",
- dest="transaction_type",
- default="CON",
- help="Comma separated list of transaction types to import",
- )
- parser.add_argument(
- "--year",
- dest="year",
- default="2023",
- help="Year to scrape",
- )
- # TODO: Add argument to optionally import from local file
-
- def handle(self, *args, **options):
- s3 = boto3.client("s3")
-
- resource_name = f"{options['transaction_type']}_{options['year']}.gz"
-
- with open(resource_name, "wb") as f:
- s3.download_fileobj(
- os.getenv("AWS_STORAGE_BUCKET_NAME", "openness-project-nmid"),
- resource_name,
- f,
- )
-
- with gzip.open(resource_name, "rt") as f:
- reader = csv.DictReader(f)
-
- key_func = lambda record: (record["OrgID"], record["Report Name"])
- sorted_records = sorted(reader, key=key_func)
-
- loans, special_events, transactions = [], [], []
-
- for filing_group, records in groupby(tqdm(sorted_records), key=key_func):
- for i, record in enumerate(records):
- if i == 0:
- filing = self.make_filing(record)
- models.LoanTransaction.objects.filter(filing=filing).delete()
- models.SpecialEvent.objects.filter(filing=filing).delete()
- models.Transaction.objects.filter(filing=filing).delete()
-
- contributor = self.make_contributor(record)
-
- if record["Contribution Type"] == "Loans Received":
- loans.append(
- self.make_contribution(record, contributor, filing)
- )
-
- elif record["Contribution Type"] == "Special Event":
- special_events.append(
- self.make_contribution(record, contributor, filing)
- )
-
- elif "Contribution" in record["Contribution Type"]:
- transactions.append(
- self.make_contribution(record, contributor, filing)
- )
-
- else:
- self.stderr.write(
- f"Could not determine contribution type from record: {record['Contribution Type']}"
- )
-
- if len(transactions) >= 2500:
- models.Transaction.objects.bulk_create(transactions)
- transactions = []
- self.stdout.write("Wrote 2500 contributions")
-
- models.LoanTransaction.objects.bulk_create(loans)
- models.SpecialEvent.objects.bulk_create(special_events)
- models.Transaction.objects.bulk_create(transactions)
-
- self.stdout.write("Wrote remaining contributions, loans, and special events")
-
- self.total_filings(options["year"])
-
- call_command("import_data", "--add-aggregates")
-
- def fetch_from_cache(self, cache_entity, cache_key, model, model_kwargs):
- try:
- return self._cache[cache_entity][cache_key]
- except KeyError:
- try:
- obj, _ = model.objects.get_or_create(**model_kwargs)
- except model.MultipleObjectsReturned:
- obj = model.objects.filter(**model_kwargs).first()
- self._cache[cache_entity][cache_key] = obj
- return obj
-
- def make_contributor(self, record):
- state = self.fetch_from_cache(
- "state",
- record["Contributor State"],
- models.State,
- {"postal_code": record["Contributor State"]},
- )
-
- address = self.fetch_from_cache(
- "address",
- (
- f"{record['Contributor Address Line 1']}{' ' + record['Contributor Address Line 2'] if record['Contributor Address Line 2'] else ''}",
- record["Contributor City"],
- state,
- record["Contributor Zip Code"],
- ),
- models.Address,
- dict(
- street=f"{record['Contributor Address Line 1']}{' ' + record['Contributor Address Line 2'] if record['Contributor Address Line 2'] else ''}",
- city=record["Contributor City"],
- state=state,
- zipcode=record["Contributor Zip Code"],
- ),
- )
-
- contact_type = self.fetch_from_cache(
- "contact_type",
- record["Contributor Code"],
- models.ContactType,
- {"description": record["Contributor Code"]},
- )
-
- full_name = re.sub(
- r"\s{2,}",
- " ",
- " ".join(
- [
- record["Prefix"],
- record["First Name"],
- record["Middle Name"],
- record["Last Name"],
- record["Suffix"],
- ]
- ),
- ).strip()
-
- contact_kwargs = {
- "prefix": record["Prefix"],
- "first_name": record["First Name"],
- "middle_name": record["Middle Name"],
- "last_name": record["Last Name"],
- "suffix": record["Suffix"],
- "occupation": record["Contributor Occupation"],
- "company_name": record["Contributor Employer"],
- "full_name": full_name,
- }
-
- try:
- contact = models.Contact.objects.get(
- **contact_kwargs,
- status_id=0,
- address=address,
- contact_type=contact_type,
- )
- except models.Contact.DoesNotExist:
- entity_type = self.fetch_from_cache(
- "entity_type",
- record["Contributor Code"][:24],
- models.EntityType,
- {"description": record["Contributor Code"][:24]},
- )
-
- entity = models.Entity.objects.create(
- user_id=self._next_entity_id,
- entity_type=entity_type,
- )
- contact = models.Contact.objects.create(
- **contact_kwargs,
- status_id=0,
- address=address,
- contact_type=contact_type,
- entity=entity,
- )
-
- self._next_entity_id += 1
-
- return contact
-
- def make_filing(self, record):
- if record["Report Entity Type"] == "Candidate":
- entity_type = self.fetch_from_cache(
- "entity_type",
- "Candidate",
- models.EntityType,
- {"description": "Candidate"},
- )
-
- entity = self.fetch_from_cache(
- "entity",
- (record["OrgID"], "Candidate"),
- models.Entity,
- {"user_id": record["OrgID"], "entity_type": entity_type},
- )
-
- full_name = re.sub(
- r"\s{2,}",
- " ",
- " ".join(
- [
- record["Candidate Prefix"],
- record["Candidate First Name"],
- record["Candidate Middle Name"],
- record["Candidate Last Name"],
- record["Candidate Suffix"],
- ]
- ),
- ).strip()
-
- # Annoyingly, some campaigns seem to have the different OrgIDs
- # for the same thing... e.g., Antonio "Moe" Maestas. We should add
- # a unique constraint to slug?
- candidate = self.fetch_from_cache(
- "candidate",
- entity.user_id,
- models.Candidate,
- dict(
- prefix=record["Candidate Prefix"],
- first_name=record["Candidate First Name"],
- middle_name=record["Candidate Middle Name"],
- last_name=record["Candidate Last Name"],
- suffix=record["Candidate Suffix"],
- full_name=full_name,
- entity=entity,
- slug=slugify(
- " ".join(
- [
- record["Candidate First Name"],
- record["Candidate Last Name"],
- ]
- )
- ),
- ),
- )
-
- election_year = parse(record["Start of Period"]).date().year
-
- election_season = self.fetch_from_cache(
- "election_season",
- (election_year, False, 0),
- models.ElectionSeason,
- dict(
- year=parse(record["Start of Period"]).date().year,
- special=False,
- status_id=0,
- ),
- )
-
- campaign = self.fetch_from_cache(
- "campaign",
- (record["Committee Name"], candidate.full_name, election_season.year),
- models.Campaign,
- dict(
- committee_name=record["Committee Name"],
- candidate=candidate,
- election_season=election_season,
- office_id=0,
- political_party_id=0,
- ),
- )
-
- filing_kwargs = {"entity": entity, "campaign": campaign}
-
- else:
- entity_type = self.fetch_from_cache(
- "entity_type",
- record["Report Entity Type"][:24],
- models.EntityType,
- {"description": record["Report Entity Type"][:24]},
- )
-
- entity = self.fetch_from_cache(
- "entity",
- (record["OrgID"], entity_type.description),
- models.Entity,
- {"user_id": record["OrgID"], "entity_type": entity_type},
- )
-
- pac = self.fetch_from_cache(
- "pac",
- entity.user_id,
- models.PAC,
- dict(
- name=record["Committee Name"],
- entity=entity,
- slug=slugify(record["Committee Name"]),
- ),
- )
-
- filing_kwargs = {"entity": entity}
-
- filing_type = self.fetch_from_cache(
- "filing_type",
- record["Report Name"][:24],
- models.FilingType,
- {"description": record["Report Name"][:24]},
- )
-
- filing_period = self.fetch_from_cache(
- "filing_period",
- (
- record["Report Name"],
- parse(record["Filed Date"]).date(),
- parse(record["Start of Period"]).date(),
- parse(record["End of Period"]).date(),
- ),
- models.FilingPeriod,
- dict(
- description=record["Report Name"],
- filing_date=parse(record["Filed Date"]).date(),
- initial_date=parse(record["Start of Period"]).date(),
- due_date=parse(record["End of Period"]).date(),
- allow_no_activity=False,
- exclude_from_cascading=False,
- email_sent_status=0,
- filing_period_type=filing_type,
- ),
- )
-
- filing = self.fetch_from_cache(
- "filing",
- (
- filing_kwargs["entity"].user_id,
- filing_period.id,
- parse(record["End of Period"]).date(),
- ),
- models.Filing,
- dict(
- filing_period=filing_period,
- date_closed=parse(record["End of Period"]).date(),
- final=True,
- **filing_kwargs,
- ),
- )
-
- return filing
-
- def make_contribution(self, record, contributor, filing):
- address_kwargs = dict(
- address=f"{record['Contributor Address Line 1']}{' ' + record['Contributor Address Line 2'] if record['Contributor Address Line 2'] else ''}",
- city=record["Contributor City"],
- state=record["Contributor State"],
- zipcode=record["Contributor Zip Code"],
- )
- if record["Contribution Type"] == "Loans Received":
- transaction_type = self.fetch_from_cache(
- "loan_transaction_type",
- "Payment",
- models.LoanTransactionType,
- {"description": "Payment"},
- )
-
- loan, _ = models.Loan.objects.get_or_create(
- amount=record["Transaction Amount"],
- received_date=parse(record["Transaction Date"]).date(),
- check_number=record["Check Number"],
- status_id=0,
- contact=contributor,
- company_name=contributor.company_name or "Not specified",
- filing=filing,
- **address_kwargs,
- )
-
- contribution = models.LoanTransaction(
- amount=record["Transaction Amount"],
- transaction_date=parse(record["Transaction Date"]).date(),
- transaction_status_id=0,
- loan=loan,
- filing=filing,
- transaction_type=transaction_type,
- )
-
- elif record["Contribution Type"] == "Special Event":
- address_kwargs.pop("state")
-
- contribution = models.SpecialEvent(
- anonymous_contributions=record["Transaction Amount"],
- event_date=parse(record["Transaction Date"]).date(),
- admission_price=0,
- attendance=0,
- total_admissions=0,
- total_expenditures=0,
- transaction_status_id=0,
- sponsors=contributor.company_name or "Not specified",
- filing=filing,
- **address_kwargs,
- )
-
- elif "Contribution" in record["Contribution Type"]:
- transaction_type = self.fetch_from_cache(
- "transaction_type",
- "Monetary contribution",
- models.TransactionType,
- {
- "description": "Monetary contribution",
- "contribution": True,
- "anonymous": False,
- },
- )
-
- contribution = models.Transaction(
- amount=record["Transaction Amount"],
- received_date=parse(record["Transaction Date"]).date(),
- check_number=record["Check Number"],
- description=record["Description"][:74],
- contact=contributor,
- full_name=contributor.full_name,
- filing=filing,
- transaction_type=transaction_type,
- **address_kwargs,
- company_name=contributor.full_name,
- occupation=record["Contributor Occupation"],
- )
-
- return contribution
-
- def total_filings(self, year):
- for filing in models.Filing.objects.filter(
- filing_period__filing_date__year=year
- ).iterator():
- contributions = filing.contributions().aggregate(total=Sum("amount"))
- expenditures = filing.expenditures().aggregate(total=Sum("amount"))
- loans = filing.loans().aggregate(total=Sum("amount"))
-
- filing.total_contributions = contributions["total"] or 0
- filing.total_expenditures = expenditures["total"] or 0
- filing.total_loans = loans["total"] or 0
-
- filing.closing_balance = filing.opening_balance or 0 + (
- filing.total_contributions
- + filing.total_loans
- - filing.total_expenditures
- )
-
- filing.save()
-
- self.stdout.write(f"Totalled {filing}")
diff --git a/camp_fin/migrations/0077_auto_20231212_1005.py b/camp_fin/migrations/0077_auto_20231212_1005.py
new file mode 100644
index 0000000..3c7ba18
--- /dev/null
+++ b/camp_fin/migrations/0077_auto_20231212_1005.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.29 on 2023-12-12 17:05
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('camp_fin', '0076_auto_20231130_1404'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='entitytype',
+ name='description',
+ field=models.CharField(max_length=256),
+ ),
+ ]
diff --git a/camp_fin/models.py b/camp_fin/models.py
index 87741c1..577aee8 100644
--- a/camp_fin/models.py
+++ b/camp_fin/models.py
@@ -129,6 +129,8 @@ def __str__(self):
self.candidate.first_name, self.candidate.last_name
)
return "{0} ({1})".format(candidate_name, office)
+ elif self.committee_name:
+ return "{0} ({1})".format(self.committee_name, office)
else:
party = self.political_party.name
return "{0} ({1})".format(party, office)
@@ -273,15 +275,16 @@ def party_identifier(self):
"""
Return a shortened version of the Campaign's party.
"""
- if self.political_party.name:
+ if self.political_party.name == "Not specified":
+ return None
+
+ elif self.political_party.name:
if self.political_party.name == "Democrat":
return "D"
elif self.political_party.name == "Republican":
return "R"
else:
return "I"
- else:
- return None
def get_status(self):
"""
@@ -752,7 +755,7 @@ class Filing(models.Model):
regenerate = models.CharField(max_length=3, null=True)
def __str__(self):
- if self.campaign:
+ if self.campaign and self.campaign.candidate:
return "{0} {1} {2}".format(
self.campaign.candidate.first_name,
self.campaign.candidate.last_name,
@@ -1058,9 +1061,19 @@ def stack_trends(trend):
)
start_month = datetime(int(since), 1, 1)
- end_month = (
- FilingPeriod.objects.order_by("-filing_date").first().filing_date.date()
- )
+
+ try:
+ end_month = (
+ self.filing_set.order_by("-filing_period__filing_date")
+ .first()
+ .filing_period.filing_date.date()
+ )
+ except:
+ end_month = (
+ FilingPeriod.objects.order_by("-filing_date")
+ .first()
+ .filing_date.date()
+ )
for month in rrule(freq=MONTHLY, dtstart=start_month, until=end_month):
replacements = {"month": month.month - 1}
@@ -1095,7 +1108,7 @@ def stack_trends(trend):
class EntityType(models.Model):
- description = models.CharField(max_length=25)
+ description = models.CharField(max_length=256)
def __str__(self):
return self.description
diff --git a/camp_fin/templates/camp_fin/candidate-detail.html b/camp_fin/templates/camp_fin/candidate-detail.html
index 2e286db..e403847 100644
--- a/camp_fin/templates/camp_fin/candidate-detail.html
+++ b/camp_fin/templates/camp_fin/candidate-detail.html
@@ -71,8 +71,8 @@ Past campaigns
{{ campaign.office.description }}
{% if campaign.party_identifier %}
-
- ({{ latest_campaign.party_identifier }})
+
+ ({{ campaign.party_identifier }})
{% endif %}
|
diff --git a/camp_fin/templates/camp_fin/search.html b/camp_fin/templates/camp_fin/search.html
index cf4af09..6517277 100644
--- a/camp_fin/templates/camp_fin/search.html
+++ b/camp_fin/templates/camp_fin/search.html
@@ -221,7 +221,7 @@ Lobbyist spending
'data': 'office_name',
'render': function(data, type, full, meta){
if (!full['office_name']) {
- return 'Not specified'
+ return ''
} else {
var name = full['office_name'];
if (full['county_name'] != '' && full['county_name'] != 'ALL'){
diff --git a/camp_fin/views.py b/camp_fin/views.py
index cba14b8..a65abff 100644
--- a/camp_fin/views.py
+++ b/camp_fin/views.py
@@ -265,7 +265,7 @@ def get_context_data(self, **kwargs):
USING(entity_id)
JOIN camp_fin_campaign AS campaign
ON filing.campaign_id = campaign.id
- LEFT JOIN camp_fin_office AS office
+ JOIN camp_fin_office AS office
ON campaign.office_id = office.id
WHERE filing.date_added >= '{year}-01-01'
AND filing.closing_balance IS NOT NULL
@@ -1219,7 +1219,7 @@ def get_context_data(self, **kwargs):
# Count pure donations, if applicable
if total_loans > 0 or total_inkind > 0:
donations = latest_filing.total_contributions - (
- latest_filing.total_loans + latest_filing.total_inkind
+ (latest_filing.total_loans or 0) + (latest_filing.total_inkind or 0)
)
context["donations"] = donations
@@ -1604,7 +1604,7 @@ def list(self, request):
ON campaign.political_party_id = party.id
LEFT JOIN camp_fin_county AS county
ON campaign.county_id = county.id
- LEFT JOIN camp_fin_office AS office
+ JOIN camp_fin_office AS office
ON campaign.office_id = office.id
LEFT JOIN camp_fin_officetype AS officetype
ON office.office_type_id = officetype.id
@@ -1937,7 +1937,7 @@ def bulk_candidates(request):
ON campaign.political_party_id = party.id
LEFT JOIN camp_fin_county AS county
ON campaign.county_id = county.id
- LEFT JOIN camp_fin_office AS office
+ JOIN camp_fin_office AS office
ON campaign.office_id = office.id
JOIN camp_fin_officetype AS officetype
ON office.office_type_id = officetype.id
diff --git a/scripts/release.sh b/scripts/release.sh
index 2fa366e..19417b4 100755
--- a/scripts/release.sh
+++ b/scripts/release.sh
@@ -6,7 +6,7 @@ python manage.py migrate --noinput
if [ `psql ${DATABASE_URL} -tAX -c "SELECT COUNT(*) FROM camp_fin_candidate"` -eq "0" ]; then
python manage.py import_data
- python -W ignore manage.py import_data_2023
+ make import/CON_2023 import/EXP_2023
python manage.py make_search_index
fi