Merge pull request #278 from dinukadesilva/gh-274

Refactoring:

migrations/env.py

@@ -47,7 +47,7 @@ def run_migrations_offline():
     """
     url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url, target_metadata=target_metadata, literal_binds=True
+        url=url, target_metadata=target_metadata, literal_binds=True, compare_type=True
     )
 
     with context.begin_transaction():
@@ -83,7 +83,8 @@ def process_revision_directives(context, revision, directives):
             connection=connection,
             target_metadata=target_metadata,
             process_revision_directives=process_revision_directives,
-            **current_app.extensions['migrate'].configure_args
+            **current_app.extensions['migrate'].configure_args,
+            compare_type=True
         )
 
         with context.begin_transaction():

migrations/versions/8b3cb0ef28e6_.py

@@ -0,0 +1,82 @@
+"""empty message
+
+Revision ID: 8b3cb0ef28e6
+Revises: 1c10a1181f81
+Create Date: 2019-11-01 01:05:21.915115
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import mysql
+
+# revision identifiers, used by Alembic.
+revision = '8b3cb0ef28e6'
+down_revision = '1c10a1181f81'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('area', 'areaName',
+               existing_type=mysql.VARCHAR(length=300),
+               type_=sa.String(length=800),
+               existing_nullable=False)
+    op.alter_column('election_candidate', 'qualifiedForPreferences',
+               existing_type=mysql.TINYINT(display_width=1),
+               type_=sa.Boolean(),
+               existing_nullable=False)
+    op.alter_column('invoice', 'confirmed',
+               existing_type=mysql.TINYINT(display_width=1),
+               type_=sa.Boolean(),
+               existing_nullable=False)
+    op.alter_column('invoice', 'delete',
+               existing_type=mysql.TINYINT(display_width=1),
+               type_=sa.Boolean(),
+               existing_nullable=True)
+    op.alter_column('invoice_stationaryItem', 'received',
+               existing_type=mysql.TINYINT(display_width=1),
+               type_=sa.Boolean(),
+               existing_nullable=False)
+    op.alter_column('proof', 'finished',
+               existing_type=mysql.TINYINT(display_width=1),
+               type_=sa.Boolean(),
+               existing_nullable=True)
+    op.alter_column('tallySheetVersion', 'isComplete',
+               existing_type=mysql.TINYINT(display_width=1),
+               type_=sa.Boolean(),
+               existing_nullable=False)
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('tallySheetVersion', 'isComplete',
+               existing_type=sa.Boolean(),
+               type_=mysql.TINYINT(display_width=1),
+               existing_nullable=False)
+    op.alter_column('proof', 'finished',
+               existing_type=sa.Boolean(),
+               type_=mysql.TINYINT(display_width=1),
+               existing_nullable=True)
+    op.alter_column('invoice_stationaryItem', 'received',
+               existing_type=sa.Boolean(),
+               type_=mysql.TINYINT(display_width=1),
+               existing_nullable=False)
+    op.alter_column('invoice', 'delete',
+               existing_type=sa.Boolean(),
+               type_=mysql.TINYINT(display_width=1),
+               existing_nullable=True)
+    op.alter_column('invoice', 'confirmed',
+               existing_type=sa.Boolean(),
+               type_=mysql.TINYINT(display_width=1),
+               existing_nullable=False)
+    op.alter_column('election_candidate', 'qualifiedForPreferences',
+               existing_type=sa.Boolean(),
+               type_=mysql.TINYINT(display_width=1),
+               existing_nullable=False)
+    op.alter_column('area', 'areaName',
+               existing_type=sa.String(length=800),
+               type_=mysql.VARCHAR(length=300),
+               existing_nullable=False)
+    ### end Alembic commands ###

orm/entities/Area/__init__.py

@@ -1,5 +1,5 @@
 from app import db
-from sqlalchemy.orm import relationship, aliased
+from sqlalchemy.orm import relationship, aliased, backref
 from sqlalchemy import and_, func, or_
 
 from orm.enums import AreaTypeEnum
@@ -10,7 +10,7 @@
 class AreaModel(db.Model):
     __tablename__ = 'area'
     areaId = db.Column(db.Integer, primary_key=True, autoincrement=True)
-    areaName = db.Column(db.String(300), nullable=False)
+    areaName = db.Column(db.String(800), nullable=False)
     areaType = db.Column(db.Enum(AreaTypeEnum), nullable=False)
     electionId = db.Column(db.Integer, db.ForeignKey(Election.Model.__table__.c.electionId), nullable=False)
     # parentAreaId = db.Column(db.Integer, db.ForeignKey(areaId), nullable=True)
@@ -23,25 +23,28 @@ class AreaModel(db.Model):
     # pollingStations = relationship("PollingStationModel")
 
     # this relationship is used for persistence
-    children = relationship("AreaModel", secondary="area_area",
-                            primaryjoin="AreaModel.areaId==AreaAreaModel.parentAreaId",
-                            secondaryjoin="AreaModel.areaId==AreaAreaModel.childAreaId"
-                            )
-    parents = relationship("AreaModel", secondary="area_area",
-                           primaryjoin="AreaModel.areaId==AreaAreaModel.childAreaId",
-                           secondaryjoin="AreaModel.areaId==AreaAreaModel.parentAreaId"
-                           )
-
-    tallySheets = relationship("TallySheetModel", secondary="submission",
-                               primaryjoin="AreaModel.areaId==SubmissionModel.areaId",
-                               secondaryjoin="SubmissionModel.submissionId==TallySheetModel.tallySheetId"
-                               )
-
-    tallySheets_PRE_41 = relationship(
-        "TallySheetModel", secondary="submission",
-        primaryjoin="AreaModel.areaId==SubmissionModel.areaId",
-        secondaryjoin="and_(SubmissionModel.submissionId==TallySheetModel.tallySheetId, TallySheetModel.tallySheetCode=='PRE_41')"
-    )
+    children = relationship("AreaAreaModel", lazy="joined",
+                            primaryjoin="AreaModel.areaId==AreaAreaModel.parentAreaId")
+
+    # children = relationship("AreaModel", secondary="area_area", lazy="subquery",
+    #                         primaryjoin="AreaModel.areaId==AreaAreaModel.parentAreaId",
+    #                         secondaryjoin="AreaModel.areaId==AreaAreaModel.childAreaId"
+    #                         )
+    # parents = relationship("AreaModel", secondary="area_area", lazy="joined",
+    #                        primaryjoin="AreaModel.areaId==AreaAreaModel.childAreaId",
+    #                        secondaryjoin="AreaModel.areaId==AreaAreaModel.parentAreaId"
+    #                        )
+    #
+    # tallySheets = relationship("TallySheetModel", secondary="submission", lazy="joined",
+    #                            primaryjoin="AreaModel.areaId==SubmissionModel.areaId",
+    #                            secondaryjoin="SubmissionModel.submissionId==TallySheetModel.tallySheetId"
+    #                            )
+    #
+    # tallySheets_PRE_41 = relationship(
+    #     "TallySheetModel", secondary="submission",
+    #     primaryjoin="AreaModel.areaId==SubmissionModel.areaId",
+    #     secondaryjoin="and_(SubmissionModel.submissionId==TallySheetModel.tallySheetId, TallySheetModel.tallySheetCode=='PRE_41')"
+    # )
 
     def __init__(self, areaName, electionId):
         super(AreaModel, self).__init__(
@@ -132,6 +135,8 @@ class AreaAreaModel(db.Model):
     parentAreaId = db.Column(db.Integer, db.ForeignKey("area.areaId"), primary_key=True)
     childAreaId = db.Column(db.Integer, db.ForeignKey("area.areaId"), primary_key=True)
 
+    #parentArea = relationship(AreaModel, foreign_keys=[parentAreaId], backref=backref('children', lazy='joined'))
+
 
 Model = AreaModel
 

orm/entities/Election/election_helper.py

@@ -118,7 +118,9 @@ def get_object(election, row, row_key, data_key=None):
         if data_store_key == "TallySheet":
             data_key = "%s-%s" % (row["TallySheet"], row["Counting Centre"])
         elif data_store_key == "Polling District":
-            data_key = "%s-%s" % (row["Polling Division"], row["Polling District"])
+            data_key = "%s-%s-%s" % (row["Electoral District"], row["Polling Division"], row["Polling District"])
+        elif data_store_key == "Counting Centre":
+            data_key = "%s-%s" % (row["Electoral District"], row["Counting Centre"])
 
         obj = get_object_from_data_store(data_key, data_store_key)
 
@@ -231,6 +233,10 @@ def get_rows_from_csv(csv_path):
         else:
             rows = []
 
+        for row in rows:
+            for cell_key in row:
+                row[cell_key] = row[cell_key].encode('unicode_escape')
+
         return rows
 
     for row in get_rows_from_csv(party_candidate_dataset_file):
@@ -254,10 +260,10 @@ def get_rows_from_csv(csv_path):
         districtCentre = get_object(root_election, row, "District Centre")
         countingCentre = get_object(ordinary_election, row, "Counting Centre")
 
-        if row["Registered Voters"] is None or len(row["Registered Voters"]) is 0:
-            registered_voters = 0
-        else:
+        try:
             registered_voters = row["Registered Voters"].replace(",", "")
+        except Exception as e:
+            registered_voters = 0
 
         pollingStation = get_object(ordinary_election, {
             "Polling Station": row["Polling Station (English)"],
@@ -320,8 +326,15 @@ def get_rows_from_csv(csv_path):
         electionCommission = get_object(root_election, {"Election Commission": "Sri Lanka Election Commission"},
                                         "Election Commission")
         districtCentre = get_object(root_election, row, "District Centre")
+
+        try:
+            registered_voters = row["Registered Voters"].replace(",", "")
+        except Exception as e:
+            registered_voters = 0
+
         countingCentre = get_object(postal_election, {
-            "Counting Centre": row["Postal Vote Counting Centre"], "Registered Voters": row["Registered Voters"]
+            "Counting Centre": row["Postal Vote Counting Centre"], "Registered Voters": registered_voters,
+            "Electoral District": row["Electoral District"]
         }, "Counting Centre")
 
         country.add_child(electoralDistrict.areaId)

schemas/__init__.py

@@ -3,6 +3,7 @@
 from app import db, ma
 from orm.entities import StationaryItem, Ballot, Invoice, BallotBox, \
     Election, Proof, Submission, Electorate, SubmissionVersion, Area, Party, BallotBook, Candidate
+from orm.entities.Area import AreaAreaModel
 from orm.entities.Election import InvalidVoteCategory, ElectionCandidate
 from orm.entities.IO import File
 from orm.entities.Invoice import InvoiceStationaryItem
@@ -99,6 +100,7 @@ class Meta:
     parties = ma.Nested(PartySchema, many=True)
     invalidVoteCategories = ma.Nested("InvalidVoteCategory_Schema", many=True)
     subElections = ma.Nested("self", only=["electionId", "electionName", "subElections", "voteType"], many=True)
+    parentElection = ma.Nested("self", only=["electionId", "electionName", "voteType"], many=True)
 
 
 class TallySheetVersionRow_PRE_41_Schema(ma.ModelSchema):
@@ -196,6 +198,7 @@ class Meta:
         # to use for deserialization
         sqla_session = db.session
 
+
 class TallySheetVersionRow_PRE_34_CO_Schema(ma.ModelSchema):
     class Meta:
         fields = (
@@ -211,6 +214,7 @@ class Meta:
         # to use for deserialization
         sqla_session = db.session
 
+
 class TallySheetVersionRow_PRE_30_PD_Schema(ma.ModelSchema):
     class Meta:
         fields = (
@@ -287,7 +291,7 @@ class Meta:
             "areaName",
             "areaType",
             "electionId",
-            #"parents",
+            # "parents",
             "children"
         )
 
@@ -299,7 +303,20 @@ class Meta:
     areaType = EnumField(AreaTypeEnum)
     electorateType = EnumField(ElectorateTypeEnum)
     parents = ma.Nested('self', only="areaId", many=True)
-    children = ma.Nested('self', only="areaId", many=True)
+    children = ma.Nested('AreaAreaSchema', only="childAreaId", many=True)
+
+
+class AreaAreaSchema(ma.ModelSchema):
+    class Meta:
+        fields = (
+            "parentAreaId",
+            "childAreaId"
+        )
+
+        model = AreaAreaModel
+        # optionally attach a Session
+        # to use for deserialization
+        sqla_session = db.session
 
 
 class AreaSchema(ma.ModelSchema):
@@ -577,6 +594,7 @@ class Meta:
     content = ma.Nested(TallySheetVersionRow_PRE_30_ED_Schema, many=True)
     areas = ma.Nested(AreaSchema, many=True)
 
+
 class TallySheetVersion_PRE_34_CO_Schema(ma.ModelSchema):
     class Meta:
         fields = (
@@ -596,6 +614,7 @@ class Meta:
     # submission = ma.Nested(SubmissionSchema)
     content = ma.Nested(TallySheetVersionRow_PRE_34_CO_Schema, many=True)
 
+
 class TallySheetVersion_PRE_30_PD_Schema(ma.ModelSchema):
     class Meta:
         fields = (