Add annual limits to dashboard and usage reports (#1989)

* feat: add client to fetch and cache annual usage for a given service

* feat(dashboard): fetch and use annual send counts on the dashboard

* chore: undo new client class

* feat(service_api_client): get annual data excluding today and cache it for use in the dashboard

* chore: formatting (why is this still happening?!?!)

* feat(dashboard): display annual data on dashboard (cached); show aggregates at top of usage report page

* chore: translations

* test(dashboard): add a mock for annual stats whenever dashboard is tested

* chore: formatting

* chore: remove duplicate translation

* chore: default `FF_ANNUAL_LIMITS` to true for staging config

* fix: only show new ui related to annual limits if `FF_ANNUAL_LIMIT` is true

* chore: update utils

* chore: use annual_limit_client to get cached stats for today

* feat: cache the result of get_monthly_notification_stats

* chore: translation

* feat(dashboard): get daily data from redis where possible and aggregate with monthly sources

* test: add testids

* chore(service_api_client): cache call

* test: add tests for dashboard and usage report changes

* chore: regen poetry lock

* fix: align with data structure in annual_limits client

* chore: bump utils

* chore: add TODO around caching for when API is updated

* chore: fix failing tests

* chore: formtting

* chore: update poetry.lock

* chore: fix tests

* debug: add logging stmt to see whats going wrong in staging

* chore: fix loggin stmt

* chore: log properly? maybe? 😱

* fix: add some missing node checks to dail limits redis structure to ensure code does error out

* fix: mistakenly trying to use redis data in "db" mode, run formatting

* fix: remove "notifications" node on redis data, as it isnt there after all

* tests: update data format to align with redis annual_limit client

* fix: use explicit timezone

* Default the FF to OFF if it isnt in the ENV vars

app/config.py

@@ -80,7 +80,7 @@ class Config(object):
     # FEATURE FLAGS
     FF_SALESFORCE_CONTACT = env.bool("FF_SALESFORCE_CONTACT", True)
     FF_RTL = env.bool("FF_RTL", True)
-    FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", True)
+    FF_ANNUAL_LIMIT = env.bool("FF_ANNUAL_LIMIT", False)
 
     FREE_YEARLY_EMAIL_LIMIT = env.int("FREE_YEARLY_EMAIL_LIMIT", 20_000_000)
     FREE_YEARLY_SMS_LIMIT = env.int("FREE_YEARLY_SMS_LIMIT", 100_000)
@@ -215,7 +215,7 @@ class Test(Development):
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
 
     FF_RTL = True
-    FF_ANNUAL_LIMIT = False
+    FF_ANNUAL_LIMIT = True
 
 
 class ProductionFF(Config):

app/extensions.py

@@ -1,5 +1,6 @@
 from flask_caching import Cache
 from notifications_utils.clients.antivirus.antivirus_client import AntivirusClient
+from notifications_utils.clients.redis.annual_limit import RedisAnnualLimit
 from notifications_utils.clients.redis.bounce_rate import RedisBounceRate
 from notifications_utils.clients.redis.redis_client import RedisClient
 from notifications_utils.clients.statsd.statsd_client import StatsdClient
@@ -10,4 +11,6 @@
 zendesk_client = ZendeskClient()
 redis_client = RedisClient()
 bounce_rate_client = RedisBounceRate(redis_client)
+annual_limit_client = RedisAnnualLimit(redis_client)
+
 cache = Cache(config={"CACHE_TYPE": "simple"})  # TODO: pull config out to config.py later

app/main/views/dashboard.py

@@ -24,7 +24,7 @@
     service_api_client,
     template_statistics_client,
 )
-from app.extensions import bounce_rate_client
+from app.extensions import annual_limit_client, bounce_rate_client
 from app.main import main
 from app.models.enum.bounce_rate_status import BounceRateStatus
 from app.models.enum.notification_statuses import NotificationStatuses
@@ -229,16 +229,83 @@ def usage(service_id):
 @main.route("/services/<service_id>/monthly")
 @user_has_permissions("view_activity")
 def monthly(service_id):
+    def combine_daily_to_annual(daily, annual, mode):
+        if mode == "redis":
+            # the redis client omits properties if there are no counts yet, so account for this here\
+            daily_redis = {
+                field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"]
+            }
+            annual["sms"] += daily_redis["sms_delivered"] + daily_redis["sms_failed"]
+            annual["email"] += daily_redis["email_delivered"] + daily_redis["email_failed"]
+        elif mode == "db":
+            annual["sms"] += daily["sms"]["requested"]
+            annual["email"] += daily["email"]["requested"]
+
+        return annual
+
+    def combine_daily_to_monthly(daily, monthly, mode):
+        if mode == "redis":
+            # the redis client omits properties if there are no counts yet, so account for this here\
+            daily_redis = {
+                field: daily.get(field, 0) for field in ["sms_delivered", "sms_failed", "email_delivered", "email_failed"]
+            }
+
+            monthly[0]["sms_counts"]["failed"] += daily_redis["sms_failed"]
+            monthly[0]["sms_counts"]["requested"] += daily_redis["sms_failed"] + daily_redis["sms_delivered"]
+            monthly[0]["email_counts"]["failed"] += daily_redis["email_failed"]
+            monthly[0]["email_counts"]["requested"] += daily_redis["email_failed"] + daily_redis["email_delivered"]
+        elif mode == "db":
+            monthly[0]["sms_counts"]["failed"] += daily["sms"]["failed"]
+            monthly[0]["sms_counts"]["requested"] += daily["sms"]["requested"]
+            monthly[0]["email_counts"]["failed"] += daily["email"]["failed"]
+            monthly[0]["email_counts"]["requested"] += daily["email"]["requested"]
+
+        return monthly
+
+    def aggregate_by_type(notification_data):
+        counts = {"sms": 0, "email": 0, "letter": 0}
+        for month_data in notification_data["data"].values():
+            for message_type, message_counts in month_data.items():
+                if isinstance(message_counts, dict):
+                    counts[message_type] += sum(message_counts.values())
+
+        # return the result
+        return counts
+
     year, current_financial_year = requested_and_current_financial_year(request)
+    monthly_data = service_api_client.get_monthly_notification_stats(service_id, year)
+    annual_data = aggregate_by_type(monthly_data)
+
+    todays_data = annual_limit_client.get_all_notification_counts(current_service.id)
+
+    # if redis is empty, query the db
+    if todays_data is None:
+        todays_data = service_api_client.get_service_statistics(service_id, limit_days=1, today_only=False)
+        annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "db")
+
+        months = (format_monthly_stats_to_list(monthly_data["data"]),)
+        monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "db")
+    else:
+        # aggregate daily + annual
+        current_app.logger.info("todays data" + str(todays_data))
+        annual_data_aggregate = combine_daily_to_annual(todays_data, annual_data, "redis")
+
+        months = (format_monthly_stats_to_list(monthly_data["data"]),)
+        monthly_data_aggregate = combine_daily_to_monthly(todays_data, months[0], "redis")
+
+    # add today's data to monthly data
+
     return render_template(
         "views/dashboard/monthly.html",
-        months=format_monthly_stats_to_list(service_api_client.get_monthly_notification_stats(service_id, year)["data"]),
+        months=monthly_data_aggregate,
         years=get_tuples_of_financial_years(
             partial_url=partial(url_for, ".monthly", service_id=service_id),
             start=current_financial_year - 2,
             end=current_financial_year,
         ),
+        annual_data=annual_data_aggregate,
         selected_year=year,
+        current_financial_year=current_financial_year,
     )
 
 
@@ -284,6 +351,21 @@ def aggregate_notifications_stats(template_statistics):
 
 
 def get_dashboard_partials(service_id):
+    def aggregate_by_type(data, daily_data):
+        counts = {"sms": 0, "email": 0, "letter": 0}
+        # flatten out this structure to match the above
+        for month_data in data["data"].values():
+            for message_type, message_counts in month_data.items():
+                if isinstance(message_counts, dict):
+                    counts[message_type] += sum(message_counts.values())
+
+        # add todays data to the annual data
+        counts = {
+            "sms": counts["sms"] + daily_data["sms"]["requested"],
+            "email": counts["email"] + daily_data["email"]["requested"],
+        }
+        return counts
+
     all_statistics_weekly = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=7)
     template_statistics_weekly = aggregate_template_usage(all_statistics_weekly)
 
@@ -300,6 +382,10 @@ def get_dashboard_partials(service_id):
     dashboard_totals_weekly = (get_dashboard_totals(stats_weekly),)
     bounce_rate_data = get_bounce_rate_data_from_redis(service_id)
 
+    # get annual data from fact table (all data this year except today)
+    annual_data = service_api_client.get_monthly_notification_stats(service_id, year=get_current_financial_year())
+    annual_data = aggregate_by_type(annual_data, dashboard_totals_daily[0])
+
     return {
         "upcoming": render_template("views/dashboard/_upcoming.html", scheduled_jobs=scheduled_jobs),
         "daily_totals": render_template(
@@ -308,6 +394,13 @@ def get_dashboard_partials(service_id):
             statistics=dashboard_totals_daily[0],
             column_width=column_width,
         ),
+        "annual_totals": render_template(
+            "views/dashboard/_totals_annual.html",
+            service_id=service_id,
+            statistics=dashboard_totals_daily[0],
+            statistics_annual=annual_data,
+            column_width=column_width,
+        ),
         "weekly_totals": render_template(
             "views/dashboard/_totals.html",
             service_id=service_id,
@@ -329,6 +422,7 @@ def get_dashboard_partials(service_id):
 
 
 def _get_daily_stats(service_id):
+    # TODO: get from redis, else fallback to template_statistics_client.get_template_statistics_for_service
     all_statistics_daily = template_statistics_client.get_template_statistics_for_service(service_id, limit_days=1)
     stats_daily = aggregate_notifications_stats(all_statistics_daily)
     dashboard_totals_daily = (get_dashboard_totals(stats_daily),)

app/notify_client/service_api_client.py

@@ -1,5 +1,5 @@
 import json
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 from flask import current_app
 from flask_login import current_user
@@ -9,6 +9,12 @@
 from app.notify_client import NotifyAdminAPIClient, _attach_current_user, cache
 
 
+def _seconds_until_midnight():
+    now = datetime.now(timezone.utc)
+    midnight = datetime.combine(now + timedelta(days=1), datetime.min.time())
+    return int((midnight - now).total_seconds())
+
+
 class ServiceAPIClient(NotifyAdminAPIClient):
     @cache.delete("user-{user_id}")
     def create_service(
@@ -377,8 +383,15 @@ def is_service_email_from_unique(self, service_id, email_from):
     def get_service_history(self, service_id):
         return self.get("/service/{0}/history".format(service_id))
 
+    # TODO: cache this once the backend is updated to exlude data from the current day
+    # @flask_cache.memoize(timeout=_seconds_until_midnight())
     def get_monthly_notification_stats(self, service_id, year):
-        return self.get(url="/service/{}/notifications/monthly?year={}".format(service_id, year))
+        return self.get(
+            url="/service/{}/notifications/monthly?year={}".format(
+                service_id,
+                year,
+            )
+        )
 
     def get_safelist(self, service_id):
         return self.get(url="/service/{}/safelist".format(service_id))
@@ -622,5 +635,15 @@ def _use_case_data_name(self, service_id):
     def _tos_key_name(self, service_id):
         return f"tos-accepted-{service_id}"
 
+    def aggregate_by_type(self, notification_data):
+        counts = {"sms": 0, "email": 0, "letter": 0}
+        for month_data in notification_data["data"].values():
+            for message_type, message_counts in month_data.items():
+                if isinstance(message_counts, dict):
+                    counts[message_type] += sum(message_counts.values())
+
+        # return the result
+        return counts
+
 
 service_api_client = ServiceAPIClient()

app/templates/views/dashboard/_totals_annual.html

@@ -0,0 +1,25 @@
+{% from "components/big-number.html" import big_number %}
+{% from "components/message-count-label.html" import message_count_label %}
+{% from 'components/remaining-messages.html' import remaining_messages %}
+{% from "components/show-more.html" import show_more %}
+
+<div class="ajax-block-container" data-testid="annual-usage">
+    <h2 class="heading-medium mt-8">
+      {{ _('Annual usage') }}
+      <br />
+      <small class="text-gray-600 text-small font-normal" style="color: #5E6975">
+        {% set current_year = current_year or (now().year if now().month < 4 else now().year + 1) %}
+        {{ _('resets on April 1, ') ~ current_year }}
+      </small> 
+    </h2>
+    <div class="grid-row contain-floats mb-10">
+      <div class="{{column_width}}">
+        {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=statistics_annual['email'], muted=true) }}
+      </div>
+      <div class="{{column_width}}">
+        {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=statistics_annual['sms'], muted=true) }}
+      </div>
+    </div>
+    {{ show_more(url_for('.monthly', service_id=current_service.id), _('Visit usage report')) }}
+</div>
+

app/templates/views/dashboard/_totals_daily.html

@@ -1,8 +1,29 @@
 {% from "components/big-number.html" import big_number %}
 {% from "components/message-count-label.html" import message_count_label %}
 {% from 'components/remaining-messages.html' import remaining_messages %}
+{% from "components/show-more.html" import show_more %}
 
-<div class="ajax-block-container">
+{% if config["FF_ANNUAL_LIMIT"] %}
+  <div class="ajax-block-container" data-testid="daily-usage">
+      <h2 class="heading-medium mt-8">
+        {{ _('Daily usage') }}
+        <br />
+        <small class="text-gray-600 text-small font-normal" style="color: #5E6975">
+          {{ _('resets at 7pm Eastern Time') }}
+        </small> 
+      </h2>
+      <div class="grid-row contain-floats mb-10">
+        <div class="{{column_width}}">
+          {{ remaining_messages(header=_('emails'), total=current_service.message_limit, used=statistics['email']['requested'], muted=true) }}
+        </div>
+        <div class="{{column_width}}">
+          {{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested'], muted=true) }}
+        </div>
+      </div>
+      {{ show_more(url_for('main.contact'), _('Request a daily limit increase')) }}  
+  </div>
+{% else %}
+  <div class="ajax-block-container">
     <h2 class="heading-medium mt-8">
       {{ _('Daily usage') }}
       <br />
@@ -18,4 +39,5 @@ <h2 class="heading-medium mt-8">
         {{ remaining_messages(header=_('text messages'), total=current_service.sms_daily_limit, used=statistics['sms']['requested']) }}
       </div>
     </div>
-</div>
+  </div>
+{% endif %}

app/templates/views/dashboard/dashboard.html

@@ -26,7 +26,10 @@ <h2 class="heading-medium mt-8">{{ _("Scheduled sends") }}</h2>
 
     {{ ajax_block(partials, updates_url, 'weekly_totals', interval=5) }}
     {{ ajax_block(partials, updates_url, 'daily_totals', interval=5) }}
-
+    {% if config["FF_ANNUAL_LIMIT"] %}
+      {{ ajax_block(partials, updates_url, 'annual_totals', interval=5) }}
+    {% endif %}
+
     <hr />
 
     {% if partials['has_template_statistics'] %}

app/templates/views/dashboard/monthly.html

@@ -1,22 +1,23 @@
-{% from "components/big-number.html" import big_number_with_status, big_number %}
+{% from "components/big-number.html" import big_number_with_status, big_number, big_number_simple %}
 {% from "components/pill.html" import pill %}
 {% from "components/table.html" import list_table, field, hidden_field_heading, right_aligned_field_heading, row_heading %}
 {% from "components/message-count-label.html" import message_count_label %}
+{% from 'components/remaining-messages.html' import remaining_messages %}
 
 {% extends "admin_template.html" %}
 
 {% block service_page_title %}
-  {{ _('Messages sent,') }}
+  {{ _('Usage report') }}
   {{ selected_year }} {{ _('to') }} {{ selected_year + 1 }} {{ _('fiscal year') }}
 {% endblock %}
 
 {% block maincolumn_content %}
 
   <h1 class="heading-large">
-    {{ _('Messages sent') }}
+    {{ _('Usage report') }}
   </h1>
 
-  <div class="mb-6">
+  <div class="mb-12">
     {{ pill(
       items=years,
       current_value=selected_year,
@@ -25,6 +26,50 @@ <h1 class="heading-large">
     ) }}
   </div>
 
+  {% if config["FF_ANNUAL_LIMIT"] %}
+    <h2 class="heading-medium mt-12">
+      {% if selected_year == current_financial_year %}
+        {{ _('Annual limit overview') }}
+      {% else %}
+        {{ _('Annual overview') }}
+      {% endif %}
+      <br />
+      <small class="text-gray-600 text-small font-normal" style="color: #5E6975">
+        {{ _('Fiscal year begins April 1, ') ~ selected_year ~ _(' and ends March 31, ') ~ (selected_year + 1) }}
+      </small> 
+    </h2>
+    <div class="grid-row contain-floats mb-10">
+      {% if selected_year == current_financial_year %}
+        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
+          {{ remaining_messages(header=_('emails'), total=current_service.email_annual_limit, used=annual_data['email']) }}
+        </div>
+        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
+          {{ remaining_messages(header=_('text messages'), total=current_service.sms_annual_limit, used=annual_data['sms']) }}
+        </div>
+      {% else %}
+        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
+          {{ big_number_simple(
+              annual_data['email'],
+              _('emails'),
+
+            )
+          }}
+        </div>
+        <div class="w-1/2 float-left py-0 px-0 px-gutterHalf box-border">
+          {{ big_number_simple(
+              annual_data['sms'],
+              _('text messages'),
+
+            )
+          }}
+        </div>
+      {% endif %}
+    </div>
+    <h2 class="heading-medium mt-12">
+      {{ _('Month by month totals') }}
+    </h2>
+  {% endif %}
+
   {% if months %}
    {% set spend_txt = _('Total spend') %}
    {% set heading_1 = _('Month') %}