Merge branch 'development' into crop-already-uploaded

src/.gitignore

@@ -13,4 +13,5 @@ deployment/aws/
 test_data/
 celerybeat-schedule.db
 *.rdb*
-venv
+*.venv
+venv

src/_main_/config/build/deployConfig.json

@@ -1,6 +1,6 @@
 {
   "IS_PROD": false,
   "IS_CANARY": false,
-  "BUILD_VERSION": "4.12.22",
-  "BUILD_VERSION_NOTES": "Feb 24: Admin portal pagination:\n- deliver content in small chunks"
+  "BUILD_VERSION": "4.13.4",
+  "BUILD_VERSION_NOTES": "Nov 11: User events nudge fix - sorting by date and 12 hour time_string"
 }

src/_main_/config/build/deployNotes.txt

@@ -1,2 +1 @@
-Feb 24: Admin portal pagination:
-- deliver content in small chunks
+Nov 11: User events nudge fix - sorting by date and 12 hour time_string

src/_main_/utils/common.py

@@ -121,24 +121,33 @@ def rename_fields(args, pairs):
 
 def serialize_all(data, full=False, **kwargs):
   #medium = (kwargs or {}).get("medium", False)
+  info = (kwargs or {}).get("info", False)
   if not data:
     return []
 
   if isinstance(data[0], dict):
     return data
 
+
   if full:
     return [d.full_json() for d in data]
+  elif info:
+    return [d.info() for d in data]
   #elif medium: 
   #  return [d.medium_json() for d in data]
   return [d.simple_json() for d in data]
 
 
-def serialize(data, full=False):
+def serialize(data, full=False, **kwargs):
+  info = (kwargs or {}).get("info", False)
   if not data:
     return {}
+
   if full:
     return data.full_json()
+  elif info: 
+    return data.info()
+
   return data.simple_json()
 
 def check_length(args, field,  min_length=5, max_length=40):
@@ -224,6 +233,12 @@ def set_cookie(response, key, value): # TODO
   response.set_cookie(key, value, MAX_AGE, samesite='Strict')
 
 
+def local_time():
+  local_zone = tz.tzlocal()
+  dt_utc = datetime.utcnow()
+  local_now = dt_utc.astimezone(local_zone)
+  return local_now
+
 
 def utc_to_local(iso_str):
   local_zone = tz.tzlocal()

src/_main_/utils/emailer/send_email.py

@@ -18,14 +18,14 @@ def is_dev_env():
     return True
 
 
-def send_massenergize_email(subject, msg, to):
+def send_massenergize_email(subject, msg, to, sender=None):
   if is_test_mode(): 
     return True
 
   message = pystmark.Message(
     subject=subject,
     to=to,
-    sender=FROM_EMAIL, 
+    sender=sender or FROM_EMAIL, 
     text=msg, 
   )
   response = pystmark.send(message, api_key=POSTMARK_EMAIL_SERVER_TOKEN)
@@ -37,13 +37,13 @@ def send_massenergize_email(subject, msg, to):
     return False
   return True
 
-def send_massenergize_email_with_attachments(temp, t_model, to, file, file_name):
+def send_massenergize_email_with_attachments(temp, t_model, to, file, file_name, sender=None):
   if is_test_mode():
     return True
   t_model = {**t_model, "is_dev":is_dev_env()}
 
 
-  message = pystmark.Message(sender=FROM_EMAIL, to=to, template_alias=temp, template_model=t_model)
+  message = pystmark.Message(sender=sender or FROM_EMAIL, to=to, template_alias=temp, template_model=t_model)
   # postmark server can be Production, Development or Testing (for local testing)
   postmark_server = POSTMARK_EMAIL_SERVER_TOKEN
   if file is not None:

src/api/constants.py

@@ -13,4 +13,13 @@
 STANDARD_USER = 'standard_user'
 GUEST_USER = 'guest_user'
 INVITED_USER = 'invited_user'  
-WHEN_USER_AUTHENTICATED_SESSION_EXPIRES = "WHEN_USER_AUTHENTICATED_SESSION_EXPIRES"
+WHEN_USER_AUTHENTICATED_SESSION_EXPIRES = "WHEN_USER_AUTHENTICATED_SESSION_EXPIRES"
+
+CSV_FIELD_NAMES = [
+    "media_url",
+    "primary_media_id",
+    "usage_stats",
+    "usage_summary",
+    "ids_of_duplicates",
+    "duplicates",
+]

src/api/handlers/community.py

@@ -250,8 +250,11 @@ def list_other_communities_for_cadmin(self, request):
   @admins_only
   def community_admin_list(self, request):  
     context: Context  = request.context
-    #args = context.get_request_body()
-    communities, err = self.service.list_communities_for_community_admin(context)
+    args: dict = context.args
+    args, err = self.validator.expect("community_ids", "str_list", is_required=False).verify(args)
+    if err:
+        return err
+    communities, err = self.service.list_communities_for_community_admin(context, args)
     if err:
       return err
     return MassenergizeResponse(data=communities)
@@ -261,6 +264,7 @@ def community_admin_list(self, request):
   def super_admin_list(self, request):
     context: Context  = request.context
     #args = context.get_request_body()
+
     communities, err = self.service.list_communities_for_super_admin(context)
     if err:
       return err

src/api/handlers/media_library.py

@@ -23,6 +23,11 @@ def registerRoutes(self):
         self.add("/gallery.item.edit", self.edit_details)
         self.add("/gallery.image.read", self.read_image)
 
+        self.add("/gallery.generate.hashes", self.generate_hashes) # A temporary route that we will need to run to generate hashes of already uploaded content (ONCE!)
+        self.add("/gallery.duplicates.summarize", self.summarize_duplicates) # Generates a CSV of duplicate images with other useful attributes
+        self.add("/gallery.duplicates.clean", self.clean_duplicates) # Allows you to clean all/some duplicates and transfer relationships to only one record
+        self.add("/gallery.duplicates.summary.print", self.print_duplicates) # Allows you to clean all/some duplicates and transfer relationships to only one record
+
     # @admins_only
     def read_image(self, request):
         """Reads the content of an image that exists in our s3 bucket and returns the base64 results as a response.
@@ -38,6 +43,61 @@ def read_image(self, request):
         if error:
             return error
         return MassenergizeResponse(data=image_data)
+
+    @admins_only
+    def print_duplicates(self, request):
+        """ Creates a downloadable file that contains the summary of duplicate media"""
+        context: Context = request.context
+        args: dict = context.args
+        self.validator.expect("type", str) #  Future Enhancement: provide type as 'csv' or 'pdf' or other formats. But currently, only CSV
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        response, error = self.service.print_duplicates(args, context)
+        if error:
+            return error
+        # return MassenergizeResponse(data=images)
+        return response
+
+    # @admins_only
+    def clean_duplicates(self, request):
+        """Based on requests params, this route can remove duplicates and re-assign relationships for a specific group of similar duplicate items, or do so for all groups """
+        context: Context = request.context
+        args: dict = context.args
+        self.validator.expect("hash", str, is_required=True)
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        images, error = self.service.clean_duplicates(args, context)
+        if error:
+            return error
+        return MassenergizeResponse(data=images)
+
+    def summarize_duplicates(self, request):
+        """Creates a summary of duplicate images and a combined list of wherever they are being used on the platform"""
+        context: Context = request.context
+        args: dict = context.args
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        images, error = self.service.summarize_duplicates(args, context)
+        if error:
+            return error
+        return MassenergizeResponse(data=images)
+
+    # @admins_only
+    def generate_hashes(self, request):
+        """Generates hashes of media images in the database that dont have hashes yet"""
+        context: Context = request.context
+        args: dict = context.args
+
+        args, err = self.validator.verify(args, strict=True)
+        if err:
+            return err
+        images, error = self.service.generate_hashes(args, context)
+        if error:
+            return error
+        return MassenergizeResponse(data=images)
 
     @admins_only
     def fetch_content(self, request):

src/api/handlers/message.py

@@ -27,6 +27,7 @@ def registerRoutes(self) -> None:
     self.add("/messages.listTeamAdminMessages", self.team_admin_list)
     self.add("/messages.replyFromCommunityAdmin", self.reply_from_community_admin)
     self.add("/messages.forwardToTeamAdmins", self.forward_to_team_admins)
+    self.add("/messages.send", self.send_message)
 
   @admins_only
   def info(self, request):
@@ -68,6 +69,23 @@ def delete(self, request):
     if err:
       return err
     return MassenergizeResponse(data=message_info)
+
+  @admins_only
+  def send_message(self, request):
+    context: Context = request.context
+    args: dict = context.args
+    self.validator.expect("id",str, is_required=False)
+    self.validator.expect("subject",str, is_required=False)
+    self.validator.expect("message",str, is_required=False)
+    self.validator.expect("sub_audience_type",str, is_required=False)
+    self.validator.expect("audience",str, is_required=False)
+    self.validator.expect("schedule",str, is_required=False)
+    self.validator.expect("community_ids",str, is_required=False)
+
+    message_info, err = self.service.send_message(context,args)
+    if err:
+      return err
+    return MassenergizeResponse(data=message_info)
 
   @admins_only
   def team_admin_list(self, request):

src/api/handlers/userprofile.py

@@ -232,7 +232,7 @@ def super_admin_list(self, request):
         args: dict = context.args
         args, err = self.validator.expect(
             "user_emails", "str_list", is_required=False
-        ).expect("community_ids", "str_list", is_required=False).verify(args)
+        ).expect("community_ids", "str_list", is_required=False).expect("user_ids", "str_list", is_required=False).verify(args)
         if err:
             return err
         users, err = self.service.list_users_for_super_admin(context, args)

src/api/services/action.py

@@ -68,8 +68,9 @@ def create_action(self, context: Context, args, user_submitted=False) -> Tuple[d
           'title': action.title,
           'body': action.featured_summary,
         }
+        # sent from MassEnergize to cadmins
         send_massenergize_rich_email(
-              subject, admin_email, 'action_submitted_email.html', content_variables)
+              subject, admin_email, 'action_submitted_email.html', content_variables, None)
 
         if IS_PROD or IS_CANARY:
           send_slack_message(

src/api/services/admin.py

@@ -31,8 +31,9 @@ def add_super_admin(self, context, args) -> Tuple[dict, MassEnergizeAPIError]:
             'admintype': 'Super',
             'admintext': "Now that you are a super admin, you have access the MassEnergize admin website at %s. You have full control over the content of our sites, can publish new communities and add new admins" % (ADMIN_URL_ROOT)
         }
+        # sent from MassEnergize to cadmins
         send_massenergize_rich_email(
-            subject, admin.email, 'new_admin_email.html', content_variables)
+            subject, admin.email, 'new_admin_email.html', content_variables, None)
         return serialize(admin, full=True), None
       except Exception as e:
         capture_message(str(e), level="error")
@@ -66,8 +67,9 @@ def add_community_admin(self, context, args) -> Tuple[dict, MassEnergizeAPIError
             'portal_link':  f"{COMMUNITY_URL_ROOT}/{res['subdomain']}",
             'admin_type': 'Community'
         }
+        #sent from MassEnergize support
         send_massenergize_rich_email(
-            subject, res["email"], 'new_admin_email.html', content_variables)
+            subject, res["email"], 'new_admin_email.html', content_variables, None)
         res["user"] = serialize(res.get("user"))
         return res, None
       except Exception as e:
@@ -112,7 +114,8 @@ def message_admin(self, context, args) -> Tuple[dict, MassEnergizeAPIError]:
             "subject": message.title,
             "message_body": message.body,
         }
-        send_massenergize_rich_email(subject, admin_email, 'contact_admin_email.html', content_variables)
+        # sent from MassEnergize to cadmins
+        send_massenergize_rich_email(subject, admin_email, 'contact_admin_email.html', content_variables, None)
 
         if IS_PROD or IS_CANARY:
           send_slack_message(

src/api/services/auth.py

@@ -3,6 +3,7 @@
 from _main_.utils.footage.spy import Spy
 from _main_.utils.common import serialize
 from _main_.utils.context import Context
+from api.utils.api_utils import get_sender_email
 from api.utils.constants import USER_EMAIL_VERIFICATION_TEMPLATE
 from firebase_admin import auth
 from _main_.utils.massenergize_errors import CustomMassenergizeError
@@ -201,8 +202,9 @@ def email_verification(self, context: Context, args):
         "community": community.name,
         "image": community.logo.file.url if community.logo.file else None
       }
+      from_email = get_sender_email(community.id)
 
-      ok = send_massenergize_email_with_attachments(USER_EMAIL_VERIFICATION_TEMPLATE,temp_data,[email], None, None)
+      ok = send_massenergize_email_with_attachments(USER_EMAIL_VERIFICATION_TEMPLATE,temp_data,[email], None, None, from_email)
       if not ok:
         return None, CustomMassenergizeError("email_not_sent")
 

src/api/services/community.py

@@ -1,11 +1,8 @@
 from sentry_sdk import capture_exception
 from _main_.utils.massenergize_errors import CustomMassenergizeError, MassEnergizeAPIError
-from _main_.utils.massenergize_response import MassenergizeResponse
 from _main_.utils.pagination import paginate
 from api.store.community import CommunityStore
 from _main_.utils.common import serialize, serialize_all
-from _main_.utils.emailer.send_email import send_massenergize_rich_email
-from _main_.utils.emailer.email_types import COMMUNITY_REGISTRATION_EMAIL
 from _main_.utils.context import Context
 from typing import Tuple
 
@@ -103,8 +100,8 @@ def list_other_communities_for_cadmin(self, context: Context) -> Tuple[list, Mas
     sorted = sort_items(communities, context.get_params())
     return paginate(sorted, context.get_pagination_data()), None
 
-  def list_communities_for_community_admin(self, context: Context) -> Tuple[list, MassEnergizeAPIError]:
-    communities, err = self.store.list_communities_for_community_admin(context)
+  def list_communities_for_community_admin(self, context: Context, args) -> Tuple[list, MassEnergizeAPIError]:
+    communities, err = self.store.list_communities_for_community_admin(context, args)
     if err:
       return None, err
     sorted = sort_items(communities, context.get_params())

src/api/services/event.py

@@ -6,6 +6,7 @@
 from _main_.utils.constants import ADMIN_URL_ROOT, COMMUNITY_URL_ROOT, ME_LOGO_PNG
 from _main_.settings import SLACK_SUPER_ADMINS_WEBHOOK_URL, IS_PROD, IS_CANARY
 from _main_.utils.emailer.send_email import send_massenergize_rich_email
+from api.utils.api_utils import get_sender_email
 from api.utils.filter_functions import sort_items
 from .utils import send_slack_message
 from api.store.utils import get_user_or_die
@@ -89,7 +90,7 @@ def rsvp_update(self, context, args) -> Tuple[dict, MassEnergizeAPIError]:
 
           # need to validate e-mails from community admins
           #from_email = community.owner_email
-          from_email = None
+          from_email = get_sender_email(event.community.id)
 
           homelink = f'{COMMUNITY_URL_ROOT}/{community.subdomain}'
 
@@ -107,6 +108,7 @@ def rsvp_update(self, context, args) -> Tuple[dict, MassEnergizeAPIError]:
             'logo': community_logo,
             'privacylink': f"{homelink}/policies?name=Privacy%20Policy"
           }
+
 
           send_massenergize_rich_email(
               subject, user_email, 'event_rsvp_email.html', content_variables, from_email)
@@ -190,8 +192,9 @@ def create_event(self, context, args, user_submitted=False) -> Tuple[dict, MassE
           'title': event.name,
           'body': event.description,
         }
+        # sent from MassEnergize to cadmins
         send_massenergize_rich_email(
-              subject, admin_email, 'event_submitted_email.html', content_variables)
+              subject, admin_email, 'event_submitted_email.html', content_variables, None)
 
         if IS_PROD or IS_CANARY:
           send_slack_message(