From 42c10192b2b3378efaebbf9adceb9a8a9bb71595 Mon Sep 17 00:00:00 2001
From: John Keiser <john@johnkeiser.com>
Date: Tue, 22 Oct 2024 12:43:06 -0700
Subject: [PATCH] Log upload stats using INFO

Also makes Redshift logger only log query status every 5 seconds to stop
spamming the log
---
 component/lambda/functions/si_logger.py            |  2 +-
 component/lambda/functions/si_redshift.py          | 14 +++++++++++---
 .../upload_billing_resource_hours_to_lago.py       |  2 +-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/component/lambda/functions/si_logger.py b/component/lambda/functions/si_logger.py
index 62773446a8..5f95bd1943 100644
--- a/component/lambda/functions/si_logger.py
+++ b/component/lambda/functions/si_logger.py
@@ -2,4 +2,4 @@
 
 
 logger = logging.getLogger()
-logging.basicConfig(level=logging.INFO)
+logger.setLevel("INFO")
diff --git a/component/lambda/functions/si_redshift.py b/component/lambda/functions/si_redshift.py
index 5b89d54f9f..aeabd106ef 100644
--- a/component/lambda/functions/si_redshift.py
+++ b/component/lambda/functions/si_redshift.py
@@ -17,6 +17,7 @@
 from botocore.client import Config
 import time
 from si_logger import logger
+import logging
 
 
 class DatabaseConnectParams(TypedDict):
@@ -54,12 +55,15 @@ def from_env(
     def __init__(
         self,
         session: boto3.Session,
+        *,
         wait_interval_seconds: float = 0.25,
+        report_interval_seconds: float = 5,
         **database_params: Unpack[DatabaseConnectParams],
     ):
         self._session = session
         self._database_params = database_params
         self._wait_interval_seconds = wait_interval_seconds
+        self._report_interval_seconds = report_interval_seconds
         self._client = self._connect()
 
     def query(self, Sql: str, **Parameters: object):
@@ -94,6 +98,8 @@ def execute(self, Sql: str, **Parameters: object):
             )
         )
 
+        last_report = time.time()
+
         while True:
             response = self.with_client(
                 lambda client: client.describe_statement(Id=statement["Id"])
@@ -110,9 +116,11 @@ def execute(self, Sql: str, **Parameters: object):
                 case "ABORTED":
                     raise Exception(f"Query aborted (Id={statement['Id']})")
 
-            logger.debug(
-                f"Query status: {status}. Waiting {self._wait_interval_seconds}s for completion... (Id={statement['Id']})"
-            )
+            if time.time() - last_report >= self._report_interval_seconds:
+                last_report = time.time()
+                logger.log(logging.INFO,
+                    f"Query status: {status}. Waiting {self._wait_interval_seconds}s for completion... (Id={statement['Id']})"
+                )
 
             time.sleep(self._wait_interval_seconds)
 
diff --git a/component/lambda/functions/upload_billing_resource_hours_to_lago.py b/component/lambda/functions/upload_billing_resource_hours_to_lago.py
index bfcd827322..c30ab1fff1 100644
--- a/component/lambda/functions/upload_billing_resource_hours_to_lago.py
+++ b/component/lambda/functions/upload_billing_resource_hours_to_lago.py
@@ -104,7 +104,7 @@ def lambda_handler(lambda_event={}, _context=None):
             last_hour_end = None
             break
 
-        logger.warning(
+        logger.info(
             f"Uploaded {uploaded_events} events in {batch_hours}-hour batch starting {-first_hour_start} hours ago!"
         )
         last_hour_end = first_hour_start if uploaded_events > 0 else None