Merge pull request #688 from dinukadesilva/fix-pe-4-response-time

Fix tally sheet list slowness

results-tabulation-api/api/ElectionApi.py

@@ -6,7 +6,7 @@
 from exception.messages import MESSAGE_CODE_ELECTION_NOT_FOUND
 from orm.entities.Election.election_helper import get_root_token
 from orm.entities import Election
-from schemas import ElectionSchema as Schema
+from schemas import ElectionSchema as Schema, AreaMapSchema
 from util import RequestBody, get_paginated_query
 
 
@@ -60,3 +60,18 @@ def create(body):
 @authorize(required_roles=[ADMIN_ROLE])
 def getRootToken(electionId):
     return get_root_token(electionId=electionId)
+
+
+@authorize(required_roles=ALL_ROLES)
+def get_area_map(electionId=None):
+    election = Election.get_by_id(electionId=electionId)
+    if election is None:
+        raise NotFoundException(
+            message="Election not found (electionId=%d)" % electionId,
+            code=MESSAGE_CODE_ELECTION_NOT_FOUND
+        )
+
+    extended_election = election.get_extended_election()
+    area_map = extended_election.get_area_map()
+
+    return AreaMapSchema(many=True).dump(area_map).data

results-tabulation-api/auth/__init__.py

@@ -183,7 +183,6 @@ def authenticate(func, *args, **kwargs):
     return func(*args, **kwargs)
 
 
-@cache.memoize(300)
 def _get_role_area_ids(parentAreaIds, areaType, voteTypes=[]):
     from orm.entities import Area, Election
 
@@ -216,12 +215,18 @@ def _get_role_area_ids(parentAreaIds, areaType, voteTypes=[]):
     return _role_area_ids
 
 
-@cache.memoize(300)
-def get_user_access_area_ids_from_claims(claims, required_roles):
+@decorator
+@authenticate
+def authorize(func, required_roles=None, *args, **kwargs):
     from orm.enums import AreaTypeEnum
 
-    user_access_area_ids = []
+    if required_roles is None:
+        return func(*args, **kwargs)
+
+    claims: Dict = get_claims()
+
     claim_found = False
+    user_access_area_ids = []
 
     for role in required_roles:
         claim = AREA_CLAIM_PREFIX + role
@@ -341,19 +346,6 @@ def get_user_access_area_ids_from_claims(claims, required_roles):
                     )
                 ])
 
-    return user_access_area_ids, claim_found
-
-
-@decorator
-@authenticate
-def authorize(func, required_roles=None, *args, **kwargs):
-    if required_roles is None:
-        return func(*args, **kwargs)
-
-    claims: Dict = get_claims()
-    user_access_area_ids, claim_found = get_user_access_area_ids_from_claims(required_roles=required_roles,
-                                                                             claims=claims)
-
     if not claim_found:
         UnauthorizedException(
             message="No matching claim found.",

results-tabulation-api/ext/ExtendedElection/ExtendedElectionParliamentaryElection2020/__init__.py

@@ -120,7 +120,8 @@ def get_area_map_query(self):
             electoral_district.c.areaName.label("electoralDistrictName"),
             counting_centre.c.areaId.label("countingCentreId"),
             counting_centre.c.areaName.label("countingCentreName"),
-            Election.Model.voteType
+            Election.Model.voteType,
+            Election.Model.electionId
         ]
 
         query_filter = [

results-tabulation-api/ext/ExtendedElection/ExtendedElectionPresidentialElection2019/__init__.py

@@ -107,7 +107,8 @@ def get_area_map_query(self):
             electoral_district.c.areaName.label("electoralDistrictName"),
             counting_centre.c.areaId.label("countingCentreId"),
             counting_centre.c.areaName.label("countingCentreName"),
-            Election.Model.voteType
+            Election.Model.voteType,
+            Election.Model.electionId
         ]
 
         query_filter = [

results-tabulation-api/ext/ExtendedElection/__init__.py

@@ -55,124 +55,17 @@ def build_election(self, party_candidate_dataset_file=None,
                        invalid_vote_categories_dataset_file=None, number_of_seats_dataset_file=None):
         pass
 
-    @cache.memoize(300)
     def get_area_map_for_tally_sheet(self, tally_sheet):
         area = tally_sheet.area
 
         return self.get_area_map(area=area)
 
-    @cache.memoize(300)
-    def get_area_map(self, area, group_by=None, filter=None):
-        from orm.enums import AreaTypeEnum
-
+    def get_area_map(self):
         area_map_subquery = self.get_area_map_query().subquery()
 
-        column_name_list = [
-            "pollingStationId", "pollingStationName",
-            "pollingDistrictId", "pollingDistrictName",
-            "countingCentreId", "countingCentreName",
-            "pollingDivisionId", "pollingDivisionName",
-            "electoralDistrictId", "electoralDistrictName",
-            "countryId", "countryName"
-        ]
-        column_name_to_column_map = {
-            "pollingStationId": area_map_subquery.c.pollingStationId,
-            "pollingStationName": area_map_subquery.c.pollingStationName,
-            "pollingDistrictId": area_map_subquery.c.pollingDistrictId,
-            "pollingDistrictName": area_map_subquery.c.pollingDistrictName,
-            "countingCentreId": area_map_subquery.c.countingCentreId,
-            "countingCentreName": area_map_subquery.c.countingCentreName,
-            "pollingDivisionId": area_map_subquery.c.pollingDivisionId,
-            "pollingDivisionName": area_map_subquery.c.pollingDivisionName,
-            "electoralDistrictId": area_map_subquery.c.electoralDistrictId,
-            "electoralDistrictName": area_map_subquery.c.electoralDistrictName,
-            "countryId": area_map_subquery.c.countryId,
-            "countryName": area_map_subquery.c.countryName
-        }
-        query_args = []
-        query_filter = []
-        query_group_by = []
-        area_and_vote_type_wise_group_by_map = {
-            AreaTypeEnum.CountingCentre: [
-                "countingCentreId",
-                "countingCentreName",
-                "pollingDivisionId",
-                "pollingDivisionName",
-                "electoralDistrictId",
-                "electoralDistrictName",
-                "countryId",
-                "countryName"
-            ],
-            AreaTypeEnum.PollingStation: [
-                "pollingDistrictId",
-                "pollingDistrictName",
-                "pollingStationId",
-                "pollingStationName",
-                "countingCentreId",
-                "countingCentreName",
-                "pollingDivisionId",
-                "pollingDivisionName",
-                "electoralDistrictId",
-                "electoralDistrictName",
-                "countryId",
-                "countryName"
-            ],
-            AreaTypeEnum.PollingDivision: [
-                "pollingDivisionId",
-                "pollingDivisionName",
-                "electoralDistrictId",
-                "electoralDistrictName",
-                "countryId",
-                "countryName"
-            ],
-            AreaTypeEnum.ElectoralDistrict: [
-                "electoralDistrictId",
-                "electoralDistrictName",
-                "countryId",
-                "countryName"
-            ],
-            AreaTypeEnum.Country: [
-                "countryId",
-                "countryName"
-            ]
-        }
-
-        area_and_vote_type_wise_filter_map = {
-            AreaTypeEnum.PollingStation: [area_map_subquery.c.pollingStationId == area.areaId],
-            AreaTypeEnum.PollingDistrict: [area_map_subquery.c.pollingDistrictId == area.areaId],
-            AreaTypeEnum.CountingCentre: [area_map_subquery.c.countingCentreId == area.areaId],
-            AreaTypeEnum.PollingDivision: [area_map_subquery.c.pollingDivisionId == area.areaId],
-            AreaTypeEnum.ElectoralDistrict: [area_map_subquery.c.electoralDistrictId == area.areaId],
-            AreaTypeEnum.Country: [area_map_subquery.c.countryId == area.areaId]
-        }
-
-        if group_by is None:
-            if area.areaType in area_and_vote_type_wise_group_by_map:
-                group_by = area_and_vote_type_wise_group_by_map[area.areaType]
-            else:
-                group_by = []
-
-        for column_name in column_name_list:
-            column = column_name_to_column_map[column_name]
-            if column_name in group_by:
-                query_group_by.append(column)
-
-                # Append the column to query.
-                query_args.append(column)
-            else:
-                query_args.append(bindparam(column_name, None))
-
-        if filter is None:
-            if area.areaType in area_and_vote_type_wise_filter_map:
-                filter = area_and_vote_type_wise_filter_map[area.areaType]
-            else:
-                filter = []
-
-        query_filter = filter
-
-        area_map = db.session.query(*query_args).filter(*query_filter).group_by(*query_group_by).all()
-
-        return area_map
+        return db.session.query(area_map_subquery).filter(
+            area_map_subquery.c.electionId.in_(self.election.get_this_and_below_election_ids())
+        )
 
     def get_area_map_query(self):
 
@@ -218,7 +111,8 @@ def get_area_map_query(self):
             polling_station.c.areaName.label("pollingStationName"),
             counting_centre.c.areaId.label("countingCentreId"),
             counting_centre.c.areaName.label("countingCentreName"),
-            Election.Model.voteType
+            Election.Model.voteType,
+            Election.Model.electionId
         ).filter(
             country__electoral_district.parentAreaId == country.c.areaId,
             country__electoral_district.childAreaId == electoral_district.c.areaId,

results-tabulation-api/orm/entities/Area/__init__.py

@@ -1,4 +1,4 @@
-from app import db, cache
+from app import db
 from sqlalchemy.orm import relationship
 from sqlalchemy import func
 
@@ -158,7 +158,6 @@ def get_associated_areas_query(areas, areaType, electionId=None):
     return db.session.query(*query_args).filter(*query_filters).group_by(*query_group_by)
 
 
-@cache.memoize(300)
 def get_associated_areas(area, areaType, electionId=None):
     result = get_associated_areas_query(areas=[area], areaType=areaType, electionId=electionId).all()
 

results-tabulation-api/orm/entities/Election/ElectionCandidate.py

@@ -14,8 +14,8 @@ class ElectionCandidateModel(db.Model):
     qualifiedForPreferences = db.Column(db.Boolean, default=False, nullable=False)
 
     election = relationship("ElectionModel", foreign_keys=[electionId])
-    party = relationship(Party.Model, foreign_keys=[partyId])
-    candidate = relationship(Candidate.Model, foreign_keys=[candidateId])
+    party = relationship(Party.Model, foreign_keys=[partyId], lazy='subquery')
+    candidate = relationship(Candidate.Model, foreign_keys=[candidateId], lazy='subquery')
 
     candidateName = association_proxy("candidate", "candidateName")
     candidateNumber = association_proxy("candidate", "candidateNumber")

results-tabulation-api/orm/entities/Election/__init__.py

@@ -23,7 +23,7 @@ class ElectionModel(db.Model):
     isListed = db.Column(db.Boolean, nullable=False, default=False)
     metaId = db.Column(db.Integer, db.ForeignKey(Meta.Model.__table__.c.metaId), nullable=True)
 
-    parties = relationship("ElectionPartyModel", order_by="ElectionPartyModel.electionPartyId")
+    parties = relationship("ElectionPartyModel", order_by="ElectionPartyModel.electionPartyId", lazy='subquery')
     _invalidVoteCategories = relationship("InvalidVoteCategoryModel")
     subElections = relationship("ElectionModel", foreign_keys=[parentElectionId])
     rootElection = relationship("ElectionModel", remote_side=[electionId], foreign_keys=[rootElectionId])

results-tabulation-api/orm/entities/Meta/__init__.py

@@ -8,7 +8,7 @@ class MetaModel(db.Model):
 
     metaId = db.Column(db.Integer, primary_key=True, autoincrement=True)
 
-    metaDataList = relationship(MetaData.Model)
+    metaDataList = relationship(MetaData.Model, lazy='subquery')
 
     @classmethod
     def create(cls, metaDataDict=None):