From 194a9aa5438574b60439ee5dbc09daedc46b5e18 Mon Sep 17 00:00:00 2001
From: Romulo Quidute Filho <116586593+rquidute@users.noreply.github.com>
Date: Mon, 18 Mar 2024 14:56:33 -0300
Subject: [PATCH] Improve download log performance (#79)

* Improve download log performance

* Remove unused import

* Fixed import sorting

* Fix file output path

* Undo last commit
---
 .../api_v1/endpoints/test_run_executions.py   |  8 ++++---
 app/log_utils.py                              | 24 ++++++++++++++++++-
 app/models/test_run_execution.py              | 12 ++++++----
 3 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/app/api/api_v1/endpoints/test_run_executions.py b/app/api/api_v1/endpoints/test_run_executions.py
index be7744b..0854410 100644
--- a/app/api/api_v1/endpoints/test_run_executions.py
+++ b/app/api/api_v1/endpoints/test_run_executions.py
@@ -324,10 +324,12 @@ def download_log(
             "Content-Disposition": f'attachment; filename="{filename}"'
         }
 
+    log_output = log_utils.convert_execution_log_to_list(
+        log=test_run_execution.log, json_entries=json_entries
+    )
+
     return StreamingResponse(
-        log_utils.log_generator(
-            log_entries=test_run_execution.log, json_entries=json_entries
-        ),
+        log_utils.async_log_generator(items=log_output),
         **options,
     )
 
diff --git a/app/log_utils.py b/app/log_utils.py
index abc2743..9f5007e 100644
--- a/app/log_utils.py
+++ b/app/log_utils.py
@@ -13,11 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import json
 from datetime import datetime
 from functools import reduce
 from io import BytesIO
 from operator import add
-from typing import Generator, List, Optional
+from typing import AsyncGenerator, Generator, List, Optional
 from zipfile import ZipFile
 
 from app import models, schemas
@@ -39,6 +40,27 @@ def log_generator(
             yield f"{log_line.level:10} | {timestamp} | {log_line.message}\n"
 
 
+async def async_log_generator(items: list) -> AsyncGenerator:
+    for log_line in items:
+        yield log_line + "\n"
+
+
+def convert_execution_log_to_list(log: list, json_entries: bool) -> list:
+    log_entries = []
+
+    for log_line in log:
+        if json_entries:
+            log_entries.append(json.dumps(log_line.__dict__))
+        else:
+            entry = log_line
+            timestamp = datetime.fromtimestamp(entry.timestamp).strftime(
+                "%Y-%m-%d %H:%M:%S.%f"
+            )
+            log_entries.append(f"{entry.level:10} | {timestamp} | {entry.message}")
+
+    return log_entries
+
+
 def group_test_run_execution_logs(
     test_run_execution: models.TestRunExecution,
 ) -> schemas.GroupedTestRunExecutionLogs:
diff --git a/app/models/test_run_execution.py b/app/models/test_run_execution.py
index cd93872..5118e53 100644
--- a/app/models/test_run_execution.py
+++ b/app/models/test_run_execution.py
@@ -20,7 +20,7 @@
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.ext.mutable import MutableList
 from sqlalchemy.ext.orderinglist import ordering_list
-from sqlalchemy.orm import Mapped, mapped_column, relationship, with_parent
+from sqlalchemy.orm import Mapped, deferred, mapped_column, relationship, with_parent
 
 from app.db.base_class import Base
 from app.db.pydantic_data_type import PydanticListType
@@ -59,10 +59,12 @@ class TestRunExecution(Base):
     test_run_config: Mapped["TestRunConfig"] = relationship(
         "TestRunConfig", back_populates="test_run_executions"
     )
-    log: Mapped[list[TestRunLogEntry]] = mapped_column(
-        MutableList.as_mutable(PydanticListType(TestRunLogEntry)),
-        default=[],
-        nullable=False,
+    log: Mapped[list[TestRunLogEntry]] = deferred(
+        mapped_column(
+            MutableList.as_mutable(PydanticListType(TestRunLogEntry)),
+            default=[],
+            nullable=False,
+        )
     )
 
     test_suite_executions: Mapped[list["TestSuiteExecution"]] = relationship(