From e195fee5c7f215a6ef5621cc31b43b77c9bc5e58 Mon Sep 17 00:00:00 2001 From: Dennis Siemensma Date: Sun, 29 Jan 2023 17:41:05 +0100 Subject: [PATCH] Revert "Disabled an automatic data migration in v5.10 until further notice #1770" This reverts commit a9c7270167a1c0af045d2b76f13b8017ce205e7a. --- docs/reference/changelog.rst | 6 - ...s_fix_total_gas_consumption_retroactive.py | 196 +++++++++--------- dsmrreader/__init__.py | 2 +- dsmrreader/provisioning/downgrade/v5.10.2.sh | 17 -- 4 files changed, 97 insertions(+), 124 deletions(-) delete mode 100644 dsmrreader/provisioning/downgrade/v5.10.2.sh diff --git a/docs/reference/changelog.rst b/docs/reference/changelog.rst index e21cebc23..7be6aa934 100644 --- a/docs/reference/changelog.rst +++ b/docs/reference/changelog.rst @@ -25,12 +25,6 @@ Current release series :depth: 1 -v5.10.1 - January 2023 ----------------------- - -- ``Fixed`` [`#1770 `_] Disabled an automatic data migration in ``v5.10`` until further notice. - - v5.10.1 - January 2023 ---------------------- diff --git a/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py b/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py index 74f8c5a1a..57373e298 100644 --- a/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py +++ b/dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py @@ -10,107 +10,103 @@ def regenerate_data(apps, schema_editor): """Try to fix any invalid gas data due to #1770.""" + DayStatistics = apps.get_model("dsmr_stats", "DayStatistics") + HourStatistics = apps.get_model("dsmr_stats", "HourStatistics") + Notification = apps.get_model("dsmr_frontend", "Notification") - # @TODO: Temporarily no-op until further notice. - return - - # DayStatistics = apps.get_model("dsmr_stats", "DayStatistics") - # HourStatistics = apps.get_model("dsmr_stats", "HourStatistics") - # Notification = apps.get_model("dsmr_frontend", "Notification") - # - # import dsmr_backend.services.backend - # - # if dsmr_backend.services.backend.is_recent_installation(): - # # Skip for new installations. - # return - # - # # No DB index, but should be fine. - # days = DayStatistics.objects.all() - # day_count = days.count() - # x = 0 - # total_difference_occurrences = 0 - # total_gas_difference = Decimal(0) - # difference_occurrences_per_year = collections.defaultdict(int) - # total_difference_per_year = collections.defaultdict(Decimal) - # per_day_messages = [] - # - # for current in days: - # x += 1 - # print( - # "Data migration: Retroactively checking/recalculating gas consumption for day statistics: {} ({}/{})".format( - # current.day, x, day_count - # ) - # ) - # - # # Straight copy from create_daily_statistics() - # hours_in_day = dsmr_backend.services.backend.hours_in_day(day=current.day) - # start_of_day = timezone.make_aware( - # timezone.datetime( - # year=current.day.year, - # month=current.day.month, - # day=current.day.day, - # hour=0, - # minute=0, - # ) - # ) - # end_of_day = start_of_day + timezone.timedelta(hours=hours_in_day) - # hours_gas_sum = HourStatistics.objects.filter( - # hour_start__gte=start_of_day, - # hour_start__lt=end_of_day, - # ).aggregate(gas_sum=Sum("gas"),)["gas_sum"] - # - # if not hours_gas_sum or not current.gas: - # continue - # - # current_day_gas_diff = hours_gas_sum - current.gas - # - # # Only override when consumption is MORE - # if current_day_gas_diff > 0: - # output_message = f"{current.day} | Before: {current.gas} m³ - Recalculated: {hours_gas_sum} m³ | Difference: {current_day_gas_diff} m³" - # print(output_message) - # per_day_messages.append(output_message) - # - # total_gas_difference += current_day_gas_diff - # total_difference_occurrences += 1 - # difference_occurrences_per_year[current.day.year] += 1 - # total_difference_per_year[current.day.year] += current_day_gas_diff - # - # # Update day, prices will be recalculated below. - # current.gas = hours_gas_sum - # current.save() - # - # if total_difference_occurrences > 0: - # # Reflect changes for prices. - # import dsmr_stats.services - # - # dsmr_stats.services.recalculate_prices() - # - # total_text = "DSMR-reader found and fixed historic gas differences in favor of issue #1770. Note that the number format is English and not localized.\n" - # - # total_text += f"\nDifferences by year ({total_gas_difference} m³ in total):\n" - # for year, total in total_difference_per_year.items(): - # total_text += f"{year} | {total} m³\n" - # - # total_text += ( - # f"\nOccurrences by year ({total_difference_occurrences} x in total):\n" - # ) - # for year, count in difference_occurrences_per_year.items(): - # total_text += f"{year} | {count} x\n" - # - # print(total_text) - # - # # Total - # Notification.objects.create(message=total_text) - # # Per-day - # per_day_messages.insert( - # 0, - # "Any per-day differences in favor of issue #1770 are displayed below. Note that the number format is English and not localized.\n", - # ) - # Notification.objects.create(message="\n".join(per_day_messages)) - # - # print( - # "The information above should also be logged as notifications, accessible on the Dashboard page of DSMR-reader." - # ) + import dsmr_backend.services.backend + + if dsmr_backend.services.backend.is_recent_installation(): + # Skip for new installations. + return + + # No DB index, but should be fine. + days = DayStatistics.objects.all() + day_count = days.count() + x = 0 + total_difference_occurrences = 0 + total_gas_difference = Decimal(0) + difference_occurrences_per_year = collections.defaultdict(int) + total_difference_per_year = collections.defaultdict(Decimal) + per_day_messages = [] + + for current in days: + x += 1 + print( + "Data migration: Retroactively checking/recalculating gas consumption for day statistics: {} ({}/{})".format( + current.day, x, day_count + ) + ) + + # Straight copy from create_daily_statistics() + hours_in_day = dsmr_backend.services.backend.hours_in_day(day=current.day) + start_of_day = timezone.make_aware( + timezone.datetime( + year=current.day.year, + month=current.day.month, + day=current.day.day, + hour=0, + minute=0, + ) + ) + end_of_day = start_of_day + timezone.timedelta(hours=hours_in_day) + hours_gas_sum = HourStatistics.objects.filter( + hour_start__gte=start_of_day, + hour_start__lt=end_of_day, + ).aggregate(gas_sum=Sum("gas"),)["gas_sum"] + + if not hours_gas_sum or not current.gas: + continue + + current_day_gas_diff = hours_gas_sum - current.gas + + # Only override when consumption is MORE + if current_day_gas_diff > 0: + output_message = f"{current.day} | Before: {current.gas} m³ - Recalculated: {hours_gas_sum} m³ | Difference: {current_day_gas_diff} m³" + print(output_message) + per_day_messages.append(output_message) + + total_gas_difference += current_day_gas_diff + total_difference_occurrences += 1 + difference_occurrences_per_year[current.day.year] += 1 + total_difference_per_year[current.day.year] += current_day_gas_diff + + # Update day, prices will be recalculated below. + current.gas = hours_gas_sum + current.save() + + if total_difference_occurrences > 0: + # Reflect changes for prices. + import dsmr_stats.services + + dsmr_stats.services.recalculate_prices() + + total_text = "DSMR-reader found and fixed historic gas differences in favor of issue #1770. Note that the number format is English and not localized.\n" + + total_text += f"\nDifferences by year ({total_gas_difference} m³ in total):\n" + for year, total in total_difference_per_year.items(): + total_text += f"{year} | {total} m³\n" + + total_text += ( + f"\nOccurrences by year ({total_difference_occurrences} x in total):\n" + ) + for year, count in difference_occurrences_per_year.items(): + total_text += f"{year} | {count} x\n" + + print(total_text) + + # Total + Notification.objects.create(message=total_text) + # Per-day + per_day_messages.insert( + 0, + "Any per-day differences in favor of issue #1770 are displayed below. Note that the number format is English and not localized.\n", + ) + Notification.objects.create(message="\n".join(per_day_messages)) + + print( + "The information above should also be logged as notifications, accessible on the Dashboard page of DSMR-reader." + ) def noop(apps, schema_editor): diff --git a/dsmrreader/__init__.py b/dsmrreader/__init__.py index cf338e86f..34b07d821 100644 --- a/dsmrreader/__init__.py +++ b/dsmrreader/__init__.py @@ -1,5 +1,5 @@ from django.utils.version import get_version -VERSION = (5, 10, 2, "final", 0) +VERSION = (5, 10, 1, "final", 0) __version__ = get_version(VERSION) diff --git a/dsmrreader/provisioning/downgrade/v5.10.2.sh b/dsmrreader/provisioning/downgrade/v5.10.2.sh deleted file mode 100644 index b6874f1df..000000000 --- a/dsmrreader/provisioning/downgrade/v5.10.2.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -# Dump for DSMR-reader v5.10.2 -./manage.py migrate dsmr_api 0003_create_api_user -./manage.py migrate dsmr_backend 0015_backend_restart_required -./manage.py migrate dsmr_backup 0017_backup_interval_in_days -./manage.py migrate dsmr_consumption 0021_schedule_quarter_hour_peaks_calculation -./manage.py migrate dsmr_datalogger 0032_dsmr_extra_device_channel -./manage.py migrate dsmr_dropbox 0001_schedule_dropbox -./manage.py migrate dsmr_frontend 0049_alter_notification_options -./manage.py migrate dsmr_influxdb 0006_influxdb_settings_field_size -./manage.py migrate dsmr_mindergas 0005_schedule_mindergas_export -./manage.py migrate dsmr_mqtt 0021_quarter_hour_peak_mqtt -./manage.py migrate dsmr_notification 0008_dummy_notification_provider -./manage.py migrate dsmr_pvoutput 0004_pvoutput_setting_refactoring -./manage.py migrate dsmr_stats 0020_day_statistics_fix_total_gas_consumption_retroactive -./manage.py migrate dsmr_weather 0006_schedule_weather_update