Skip to content

Commit

Permalink
Disabled an automatic data migration in v5.10 until further notice #1770
Browse files Browse the repository at this point in the history
  • Loading branch information
@dennissiemensma
dennissiemensma committed Jan 29, 2023
1 parent a9a411e commit a9c7270
Show file tree
Hide file tree
Showing 4 changed files with 124 additions and 97 deletions.
6 changes: 6 additions & 0 deletions docs/reference/changelog.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ Current release series
:depth: 1


v5.10.1 - January 2023
----------------------

- ``Fixed`` [`#1770 <https://github.com/dsmrreader/dsmr-reader/issues/1770>`_] Disabled an automatic data migration in ``v5.10`` until further notice.


v5.10.1 - January 2023
----------------------

Expand Down
196 changes: 100 additions & 96 deletions dsmr_stats/migrations/0020_day_statistics_fix_total_gas_consumption_retroactive.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,103 +10,107 @@

def regenerate_data(apps, schema_editor):
"""Try to fix any invalid gas data due to #1770."""
DayStatistics = apps.get_model("dsmr_stats", "DayStatistics")
HourStatistics = apps.get_model("dsmr_stats", "HourStatistics")
Notification = apps.get_model("dsmr_frontend", "Notification")

import dsmr_backend.services.backend

if dsmr_backend.services.backend.is_recent_installation():
# Skip for new installations.
return

# No DB index, but should be fine.
days = DayStatistics.objects.all()
day_count = days.count()
x = 0
total_difference_occurrences = 0
total_gas_difference = Decimal(0)
difference_occurrences_per_year = collections.defaultdict(int)
total_difference_per_year = collections.defaultdict(Decimal)
per_day_messages = []

for current in days:
x += 1
print(
"Data migration: Retroactively checking/recalculating gas consumption for day statistics: {} ({}/{})".format(
current.day, x, day_count
)
)

# Straight copy from create_daily_statistics()
hours_in_day = dsmr_backend.services.backend.hours_in_day(day=current.day)
start_of_day = timezone.make_aware(
timezone.datetime(
year=current.day.year,
month=current.day.month,
day=current.day.day,
hour=0,
minute=0,
)
)
end_of_day = start_of_day + timezone.timedelta(hours=hours_in_day)
hours_gas_sum = HourStatistics.objects.filter(
hour_start__gte=start_of_day,
hour_start__lt=end_of_day,
).aggregate(gas_sum=Sum("gas"),)["gas_sum"]

if not hours_gas_sum or not current.gas:
continue

current_day_gas_diff = hours_gas_sum - current.gas

# Only override when consumption is MORE
if current_day_gas_diff > 0:
output_message = f"{current.day} | Before: {current.gas} m³ - Recalculated: {hours_gas_sum} m³ | Difference: {current_day_gas_diff} m³"
print(output_message)
per_day_messages.append(output_message)

total_gas_difference += current_day_gas_diff
total_difference_occurrences += 1
difference_occurrences_per_year[current.day.year] += 1
total_difference_per_year[current.day.year] += current_day_gas_diff

# Update day, prices will be recalculated below.
current.gas = hours_gas_sum
current.save()

if total_difference_occurrences > 0:
# Reflect changes for prices.
import dsmr_stats.services

dsmr_stats.services.recalculate_prices()

total_text = "DSMR-reader found and fixed historic gas differences in favor of issue #1770. Note that the number format is English and not localized.\n"

total_text += f"\nDifferences by year ({total_gas_difference} m³ in total):\n"
for year, total in total_difference_per_year.items():
total_text += f"{year} | {total} m³\n"

total_text += (
f"\nOccurrences by year ({total_difference_occurrences} x in total):\n"
)
for year, count in difference_occurrences_per_year.items():
total_text += f"{year} | {count} x\n"

print(total_text)

# Total
Notification.objects.create(message=total_text)
# Per-day
per_day_messages.insert(
0,
"Any per-day differences in favor of issue #1770 are displayed below. Note that the number format is English and not localized.\n",
)
Notification.objects.create(message="\n".join(per_day_messages))

print(
"The information above should also be logged as notifications, accessible on the Dashboard page of DSMR-reader."
)
# @TODO: Temporarily no-op until further notice.
return

# DayStatistics = apps.get_model("dsmr_stats", "DayStatistics")
# HourStatistics = apps.get_model("dsmr_stats", "HourStatistics")
# Notification = apps.get_model("dsmr_frontend", "Notification")
#
# import dsmr_backend.services.backend
#
# if dsmr_backend.services.backend.is_recent_installation():
# # Skip for new installations.
# return
#
# # No DB index, but should be fine.
# days = DayStatistics.objects.all()
# day_count = days.count()
# x = 0
# total_difference_occurrences = 0
# total_gas_difference = Decimal(0)
# difference_occurrences_per_year = collections.defaultdict(int)
# total_difference_per_year = collections.defaultdict(Decimal)
# per_day_messages = []
#
# for current in days:
# x += 1
# print(
# "Data migration: Retroactively checking/recalculating gas consumption for day statistics: {} ({}/{})".format(
# current.day, x, day_count
# )
# )
#
# # Straight copy from create_daily_statistics()
# hours_in_day = dsmr_backend.services.backend.hours_in_day(day=current.day)
# start_of_day = timezone.make_aware(
# timezone.datetime(
# year=current.day.year,
# month=current.day.month,
# day=current.day.day,
# hour=0,
# minute=0,
# )
# )
# end_of_day = start_of_day + timezone.timedelta(hours=hours_in_day)
# hours_gas_sum = HourStatistics.objects.filter(
# hour_start__gte=start_of_day,
# hour_start__lt=end_of_day,
# ).aggregate(gas_sum=Sum("gas"),)["gas_sum"]
#
# if not hours_gas_sum or not current.gas:
# continue
#
# current_day_gas_diff = hours_gas_sum - current.gas
#
# # Only override when consumption is MORE
# if current_day_gas_diff > 0:
# output_message = f"{current.day} | Before: {current.gas} m³ - Recalculated: {hours_gas_sum} m³ | Difference: {current_day_gas_diff} m³"
# print(output_message)
# per_day_messages.append(output_message)
#
# total_gas_difference += current_day_gas_diff
# total_difference_occurrences += 1
# difference_occurrences_per_year[current.day.year] += 1
# total_difference_per_year[current.day.year] += current_day_gas_diff
#
# # Update day, prices will be recalculated below.
# current.gas = hours_gas_sum
# current.save()
#
# if total_difference_occurrences > 0:
# # Reflect changes for prices.
# import dsmr_stats.services
#
# dsmr_stats.services.recalculate_prices()
#
# total_text = "DSMR-reader found and fixed historic gas differences in favor of issue #1770. Note that the number format is English and not localized.\n"
#
# total_text += f"\nDifferences by year ({total_gas_difference} m³ in total):\n"
# for year, total in total_difference_per_year.items():
# total_text += f"{year} | {total} m³\n"
#
# total_text += (
# f"\nOccurrences by year ({total_difference_occurrences} x in total):\n"
# )
# for year, count in difference_occurrences_per_year.items():
# total_text += f"{year} | {count} x\n"
#
# print(total_text)
#
# # Total
# Notification.objects.create(message=total_text)
# # Per-day
# per_day_messages.insert(
# 0,
# "Any per-day differences in favor of issue #1770 are displayed below. Note that the number format is English and not localized.\n",
# )
# Notification.objects.create(message="\n".join(per_day_messages))
#
# print(
# "The information above should also be logged as notifications, accessible on the Dashboard page of DSMR-reader."
# )


def noop(apps, schema_editor):
Expand Down
2 changes: 1 addition & 1 deletion dsmrreader/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from django.utils.version import get_version

VERSION = (5, 10, 1, "final", 0)
VERSION = (5, 10, 2, "final", 0)

__version__ = get_version(VERSION)
17 changes: 17 additions & 0 deletions dsmrreader/provisioning/downgrade/v5.10.2.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env bash

# Dump for DSMR-reader v5.10.2
./manage.py migrate dsmr_api 0003_create_api_user
./manage.py migrate dsmr_backend 0015_backend_restart_required
./manage.py migrate dsmr_backup 0017_backup_interval_in_days
./manage.py migrate dsmr_consumption 0021_schedule_quarter_hour_peaks_calculation
./manage.py migrate dsmr_datalogger 0032_dsmr_extra_device_channel
./manage.py migrate dsmr_dropbox 0001_schedule_dropbox
./manage.py migrate dsmr_frontend 0049_alter_notification_options
./manage.py migrate dsmr_influxdb 0006_influxdb_settings_field_size
./manage.py migrate dsmr_mindergas 0005_schedule_mindergas_export
./manage.py migrate dsmr_mqtt 0021_quarter_hour_peak_mqtt
./manage.py migrate dsmr_notification 0008_dummy_notification_provider
./manage.py migrate dsmr_pvoutput 0004_pvoutput_setting_refactoring
./manage.py migrate dsmr_stats 0020_day_statistics_fix_total_gas_consumption_retroactive
./manage.py migrate dsmr_weather 0006_schedule_weather_update

0 comments on commit a9c7270

Please sign in to comment.