add pre-commit to clean up code (#503)

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.7.1
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format

__mocks__/passage-user.ts

@@ -9,4 +9,4 @@ jest.mock('@passageidentity/passage-elements/passage-user', () => {
     PassageUser: jest.fn(() => {}) ,
     PassageUserInfo:  jest.fn(() => {}) ,
   };
-});
+});

api/db/migrations/20240328182356_add_upload_validation_constraint/migration.sql

@@ -1 +1 @@
-CREATE UNIQUE INDEX "UploadValidation_unique_null_results_per_uploadid" ON "UploadValidation"("uploadId") WHERE results IS NULL;
+CREATE UNIQUE INDEX "UploadValidation_unique_null_results_per_uploadid" ON "UploadValidation"("uploadId") WHERE results IS NULL;

api/db/migrations/20240603182838_populate_organization_preferences/migration.sql

@@ -2,12 +2,12 @@ BEGIN;
 
 CREATE OR REPLACE FUNCTION populate_organization_preferences()
 RETURNS void AS $$
-DECLARE 
+DECLARE
     current_reporting_period_id INT;
-BEGIN 
+BEGIN
     -- Get one valid reporting period id --
-    SELECT id INTO current_reporting_period_id 
-    FROM "ReportingPeriod" 
+    SELECT id INTO current_reporting_period_id
+    FROM "ReportingPeriod"
     WHERE "outputTemplateId" IS NOT NULL AND "inputTemplateId" IS NOT NULL
     LIMIT 1;
 
@@ -27,4 +27,4 @@ $$ LANGUAGE plpgsql;
 
 SELECT populate_organization_preferences();
 
-COMMIT;
+COMMIT;

api/db/migrations/migration_lock.toml

@@ -1,3 +1,3 @@
 # Please do not edit this file manually
 # It should be added in your version-control system (i.e. Git)
-provider = "postgresql"
+provider = "postgresql"

api/db/schema.prisma

@@ -141,7 +141,7 @@ model UploadValidation {
   upload              Upload        @relation(fields: [uploadId], references: [id], onDelete: Cascade)
   results             Json?         @db.JsonB
   passed              Boolean
-  isManual            Boolean       @default(false)   
+  isManual            Boolean       @default(false)
   initiatedById       Int
   initiatedBy         User          @relation(fields: [initiatedById], references: [id])
   createdAt           DateTime      @default(now()) @db.Timestamptz(6)
@@ -169,7 +169,7 @@ enum SubrecipientStatus {
 
 model SubrecipientUpload {
   id                  Int               @id @default(autoincrement())
-  subrecipientId      Int                 
+  subrecipientId      Int
   subrecipient        Subrecipient      @relation(fields: [subrecipientId], references: [id])
   uploadId            Int
   upload              Upload            @relation(fields: [uploadId], references: [id])

api/src/functions/processValidationJson/processValidationJson.ts

@@ -61,14 +61,14 @@ export const handler = async (
     /*
     This allows us to easily test this function during local development by making a GET request as follows:
     http://localhost:8911/processValidationJson?Records[0][s3][bucket][name]=cpf-reporter&Records[0][s3][object][key]=/uploads/1/2/3/14/test.json
-    {  
-      "Records":[  
-          {  
+    {
+      "Records":[
+          {
             "s3":{
-                "bucket":{  
+                "bucket":{
                   "name":"cpf-reporter"
                 },
-                "object":{  
+                "object":{
                   "key":"/uploads/1/2/3/4/test.json"
                 }
             }

docker-compose.dev.yml

@@ -61,7 +61,7 @@ services:
       - TEST_DATABASE_URL=postgresql://redwood:redwood@db:5432/redwood_test
     depends_on:
       - db
-  
+
   # docker compose -f ./docker-compose.dev.yml run --rm -it python-console /bin/bash
   python-console:
     user: root

docs/python-schema.md

@@ -1,7 +1,7 @@
 # Python Schema for Excel CPF Templates
 
 In order to validate user reporting templates, we define a schema in Python that matches the Excel template schema (tabs, columns, and field types).
-This schema is defined and validated using Pydantic, a Python library for data validation (docs [here](https://docs.pydantic.dev/latest/)). 
+This schema is defined and validated using Pydantic, a Python library for data validation (docs [here](https://docs.pydantic.dev/latest/)).
 
 ## Note on terminology
 For the purposes of this guide, we will use `file` to refer to code files, and `sheet` to refer to user-uploaded Excel reporting templates.
@@ -22,7 +22,7 @@ For example, in a `schema.py` file:
         ..., serialization_alias="Project Name", json_schema_extra={"column":"C"}
     )
 ```
-There should only be _one_ field per column per tab (e.g., the "Project" tab can only ever have a `Project_Name__c` field in column C). 
+There should only be _one_ field per column per tab (e.g., the "Project" tab can only ever have a `Project_Name__c` field in column C).
 
 ## Updating the schema
 

python/src/functions/create_archive.py

@@ -49,7 +49,10 @@ def handle(event: dict[str, Any], _context: Context):
         create_archive(organization_id, reporting_period_id, boto3.client("s3"), logger)
     except Exception:
         logger.exception("Exception creating archive")
-        return {"statusCode": 500, "body": "Internal Server Error - unable to create archive"}
+        return {
+            "statusCode": 500,
+            "body": "Internal Server Error - unable to create archive",
+        }
 
     return {
         "statusCode": 200,

python/src/functions/generate_presigned_url_and_send_email.py

@@ -21,6 +21,7 @@
 Your treasury report can be downloaded here: {url}.
 """
 
+
 class SendTreasuryEmailLambdaPayload(BaseModel):
     organization: OrganizationObj
     user: UserObj
@@ -34,7 +35,7 @@ def handle(event: SendTreasuryEmailLambdaPayload, context: Context):
     contains a pre-signed URL to the following S3 object if it exists:
     treasuryreports/{organization.id}/{organization.preferences.current_reporting_period_id}/report.zip
     If the object does not exist then raise an exception.
-    
+
     Args:
         event: S3 Lambda event of type `s3:ObjectCreated:*`
         context: Lambda context
@@ -59,33 +60,37 @@ def handle(event: SendTreasuryEmailLambdaPayload, context: Context):
 
 
 def generate_email(
-        user: UserObj,
-        logger: structlog.stdlib.BoundLogger,
-        presigned_url: str = "",
+    user: UserObj,
+    logger: structlog.stdlib.BoundLogger,
+    presigned_url: str = "",
 ) -> Tuple[Optional[str], Optional[str], Optional[str]]:
     try:
         with open("src/static/email_templates/formatted_body.html") as g:
-            email_body = chevron.render(g, {
-                "body_title": 'Hello,',
-                "body_detail": treasury_email_html.format(
-                    url = presigned_url
-                ),
-            })
-            with open("src/static/email_templates/base.html") as f:
-                email_html = chevron.render(f, {
-                    "tool_name": "CPF",
-                    "title": "CPF Treasury Report",
-                    "preheader": False,
-                    "webview_available": False,
-                    "base_url_safe": "",
-                    "usdr_logo_url": 'https://grants.usdigitalresponse.org/usdr_logo_transparent.png',
-                    "presigned_url": presigned_url,
-                    "notifications_url_safe": False,
-                    "email_body": email_body,
+            email_body = chevron.render(
+                g,
+                {
+                    "body_title": "Hello,",
+                    "body_detail": treasury_email_html.format(url=presigned_url),
                 },
-                partials_dict = {
-                    "email_body": email_body,
-                })
+            )
+            with open("src/static/email_templates/base.html") as f:
+                email_html = chevron.render(
+                    f,
+                    {
+                        "tool_name": "CPF",
+                        "title": "CPF Treasury Report",
+                        "preheader": False,
+                        "webview_available": False,
+                        "base_url_safe": "",
+                        "usdr_logo_url": "https://grants.usdigitalresponse.org/usdr_logo_transparent.png",
+                        "presigned_url": presigned_url,
+                        "notifications_url_safe": False,
+                        "email_body": email_body,
+                    },
+                    partials_dict={
+                        "email_body": email_body,
+                    },
+                )
                 email_text = treasury_email_text.format(url=presigned_url)
                 subject = "USDR CPF Treasury Report"
                 return email_html, email_text, subject
@@ -95,8 +100,8 @@ def generate_email(
 
 
 def process_event(
-        payload: SendTreasuryEmailLambdaPayload,
-        logger: structlog.stdlib.BoundLogger,
+    payload: SendTreasuryEmailLambdaPayload,
+    logger: structlog.stdlib.BoundLogger,
 ):
     """
     This function is structured as followed:
@@ -110,20 +115,20 @@ def process_event(
     s3_client = boto3.client("s3")
     user = payload.user
     organization = payload.organization
-    
+
     presigned_url = get_presigned_url(
         s3_client=s3_client,
         bucket=os.environ["REPORTING_DATA_BUCKET_NAME"],
         key=f"treasuryreports/{organization.id}/{organization.preferences.current_reporting_period_id}/report.zip",
         expiration_time=60 * 60,  # 1 hour
     )
     if presigned_url is None:
-        raise Exception('Failed to generate signed-URL or file not found')
+        raise Exception("Failed to generate signed-URL or file not found")
 
     email_html, email_text, subject = generate_email(
-        user = user,
-        presigned_url = presigned_url,
-        logger = logger,
+        user=user,
+        presigned_url=presigned_url,
+        logger=logger,
     )
     if not email_html:
         return False

python/src/functions/subrecipient_treasury_report_gen.py

@@ -165,6 +165,7 @@ def get_recent_subrecipients(
 
     return recent_subrecipients
 
+
 def no_subrecipients_in_file(recent_subrecipients: dict):
     """
     Helper method to determine if the recent_subrecipients JSON object in

python/src/functions/test_lambdas.py

@@ -1,6 +1,7 @@
 """
 A collection of functions used for lambda testing
 """
+
 from typing import Any
 
 from aws_lambda_typing.context import Context
@@ -20,5 +21,5 @@ def pre_create_archive(event: dict[str, Any], _context: Context) -> dict[str, An
         "Payload": {
             "organization_id": "test_org_id",
             "reporting_period_id": "test_reporting_period_id",
-        }
+        },
     }

python/src/lib/email.py

@@ -7,18 +7,14 @@
 
 
 def send_email(
-        dest_email: str,
-        email_html: str,
-        email_text: str,
-        subject: str,
-        logger
-)  -> bool:
+    dest_email: str, email_html: str, email_text: str, subject: str, logger
+) -> bool:
     # Email user
     email_client = boto3.client("ses")
 
     # Try to send the email.
     try:
-        #Provide the contents of the email.
+        # Provide the contents of the email.
         response = email_client.send_email(
             Destination={
                 "ToAddresses": [
@@ -43,7 +39,7 @@ def send_email(
             },
             Source=os.getenv("NOTIFICATIONS_EMAIL"),
         )
-    # Display an error if something goes wrong.	
+    # Display an error if something goes wrong.
     except ClientError as e:
         error = e.response.get("Error") or {}
         message = error.get("Message")

python/src/lib/s3_helper.py

@@ -60,10 +60,10 @@ def upload_generated_file_to_s3(
 
 
 def get_presigned_url(
-        s3_client: S3Client,
-        bucket: str,
-        key: str,
-        expiration_time: int = 60 * 60,  #  1 hour
+    s3_client: S3Client,
+    bucket: str,
+    key: str,
+    expiration_time: int = 60 * 60,  #  1 hour
 ) -> Optional[str]:
     logger = get_logger()
     try:
@@ -76,16 +76,16 @@ def get_presigned_url(
         return None
 
     try:
-        response =  s3_client.generate_presigned_url(
+        response = s3_client.generate_presigned_url(
             "get_object",
             Params={
                 "Bucket": bucket,
                 "Key": key,
             },
-            ExpiresIn=expiration_time
+            ExpiresIn=expiration_time,
         )
     except ClientError:
         logger.exception(f"Unable to retrieve presigned URL for key: {key}")
         return None
-    
+
     return response

python/src/lib/treasury_generation_common.py

@@ -24,6 +24,7 @@ class UserObj(BaseModel):
     id: int
     email: str
 
+
 class OutputFileType(Enum):
     XLSX = "xlsx"
     CSV = "csv"

python/tests/test_create_archive.bats

@@ -59,7 +59,7 @@ teardown_file() {
   [ "$status" -eq 0 ]
   # make sure report.zip is in output
 
-  
+
 
 
 }

scripts/local_treasury_file_generation/clear_group.sh

@@ -2,7 +2,7 @@
 # Note: you will need to replace the GROUP variable with the log group name that you are interested in.
 export GROUP=/aws/lambda/localstack-lambda-project-1C
 for STREAM in $(awslocal logs describe-log-streams --log-group-name $GROUP  --query logStreams[*].logStreamName  --output text --order-by LastEventTime --descending) ;
-do  
+do
     echo awslocal logs delete-log-stream --log-group-name $GROUP  --log-stream-name $STREAM
     echo awslocal logs get-log-events --log-group-name $GROUP  --log-stream-name $STREAM --output table
 done

scripts/sample_lambda.py

@@ -1,6 +1,8 @@
 """
- To build this, zip it using zip scripts/function.zip scripts/sample_lambda.py
+To build this, zip it using zip scripts/function.zip scripts/sample_lambda.py
 """
+
+
 def lambda_handler(event, context):
     print(event)
     return {k: f"{v}_sample" for k, v in event["input"].items()}

terraform/datadog_metrics.tf

@@ -44,4 +44,4 @@ resource "datadog_metric_metadata" "custom" {
   per_unit    = each.value.per_unit
 
   depends_on = [data.http.datadog_metric_metadata]
-}
+}

terraform/variables.tf

@@ -325,4 +325,3 @@ variable "notifications_email_address" {
     error_message = "Email address must contain exactly one @ sign."
   }
 }
-

web/src/components/Organization/NewOrganizationForm/NewOrganizationForm.tsx

@@ -18,8 +18,8 @@ interface NewOrganizationFormProps {
   loading: boolean
 }
 
-/* 
-  This form creates a new organization, assigns a new agency to that organization, 
+/*
+  This form creates a new organization, assigns a new agency to that organization,
   and assigns a new admin user to both the organization and the agency.
 */
 const NewOrganizationForm = (props: NewOrganizationFormProps) => {