Merge pull request #2718 from ziv17/2699-street-cache

Improve street cache update time

alembic/versions/e962054e4422_add_street_numeric_indexes_to_involved_.py

@@ -0,0 +1,28 @@
+"""add street numeric indexes to involved heberw
+
+Revision ID: e962054e4422
+Revises: mn9px8cacn24
+Create Date: 2024-10-20 08:24:08.746964
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'e962054e4422'
+down_revision = 'mn9px8cacn24'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+# import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_index('ix_involved_markers_hebrew_street1',
+                    'involved_markers_hebrew', ['street1'], unique=False)
+    op.create_index('ix_involved_markers_hebrew_street2',
+                    'involved_markers_hebrew', ['street2'], unique=False)
+
+
+def downgrade():
+    op.drop_index('ix_involved_markers_hebrew_street1', 'involved_markers_hebrew')
+    op.drop_index('ix_involved_markers_hebrew_street2', 'involved_markers_hebrew')

anyway/parsers/infographics_data_cache_updater.py

@@ -179,46 +179,45 @@ def get_streets() -> Iterable[Streets]:
 
 def get_street_infographic_keys() -> Iterable[Dict[str, int]]:
     for street in get_streets():
-        for y in CONST.INFOGRAPHICS_CACHE_YEARS_AGO:
-            yield {
-                "yishuv_symbol": street.yishuv_symbol,
-                "street1": street.street,
-                "years_ago": y,
-                "lang": "en",
-            }
+        if street_has_accidents(street.yishuv_symbol, street.street):
+            for y in CONST.INFOGRAPHICS_CACHE_YEARS_AGO:
+                yield {
+                    "yishuv_symbol": street.yishuv_symbol,
+                    "street1": street.street,
+                    "years_ago": y,
+                    "lang": "en",
+                }
 
 
 def build_street_cache_into_temp():
     start = datetime.now()
     db.session.query(InfographicsStreetDataCacheTemp).delete()
     db.session.commit()
-    for n, chunk in enumerate(chunked_generator(get_street_infographic_keys(), 4960)):
-        cache_chunk = [
-            {
-                "yishuv_symbol": d["yishuv_symbol"],
-                "street": d["street1"],
-                "years_ago": d["years_ago"],
-                "data": anyway.infographics_utils.create_infographics_data_for_location(d),
-            }
-            for d in chunk
-            if street_has_accidents(d["yishuv_symbol"], d["street1"])
-        ]
-        if cache_chunk:
-            logging.debug(f"Adding chunk num {n}, {len(chunk)} entries.")
-            # pylint: disable=no-member
-            db.get_engine().execute(InfographicsStreetDataCacheTemp.__table__.insert(), cache_chunk)
+    for chunk in chunked_generator(get_street_infographic_keys(), 4960):
+        db.get_engine().execute(
+            InfographicsStreetDataCacheTemp.__table__.insert(),
+            [
+                {
+                    "yishuv_symbol": d["yishuv_symbol"],
+                    "street": d["street1"],
+                    "years_ago": d["years_ago"],
+                    "data": anyway.infographics_utils.create_infographics_data_for_location(d),
+                }
+                for d in chunk
+            ],
+        )
     db.session.commit()
     logging.info(f"cache rebuild took:{str(datetime.now() - start)}")
 
 
 def street_has_accidents(yishuv_symbol: int, street: int) -> bool:
-    return (
+    res = db.session.query(
         db.session.query(AccidentMarker)
         .filter(AccidentMarker.yishuv_symbol == yishuv_symbol)
         .filter(or_(AccidentMarker.street1 == street, AccidentMarker.street2 == street))
-        .count()
-        > 0
-    )
+        .exists()
+    ).scalar()
+    return res
 
 
 def get_road_segments() -> Iterable[RoadSegments]:

anyway/widgets/widget.py

@@ -30,7 +30,7 @@ class Widget:
     def __init__(self, request_params: RequestParams):
         self.request_params = copy.deepcopy(request_params)
         self.rank = -1
-        self.items = {}
+        self.items = None
         self.text = {}
         self.meta = {"widget_digest": self.widget_digest}
         self.information = ""
@@ -53,7 +53,7 @@ def is_included(self) -> bool:
 
     def generate_items(self) -> None:
         """Generates the data of the widget and set it to self.items"""
-        pass
+        self.items = {}
 
     @staticmethod
     def is_relevant(request_params: RequestParams) -> bool:
@@ -104,7 +104,7 @@ def generate_widget_data(cls, request_params: RequestParams):
         return {}
 
     def serialize(self):
-        if not self.items:
+        if self.items is None:
             self.generate_items()
         output = {"name": self.name, "data": {}}
         output["data"]["items"] = self.items if self.is_included() else {}