Merge branch 'main' into dependabot/pip/virtualenv-20.27.0

importing/tasks.py

@@ -1,6 +1,8 @@
 from logging import getLogger
 
-from huey.contrib.djhuey import on_commit_task
+from huey.contrib.djhuey import on_commit_task, task
+
+from station.functions import update_variables_for_station
 
 from .functions import save_temp_data_to_permanent
 from .models import DataImport
@@ -26,6 +28,7 @@ def ingest_data(data_import_pk: int) -> None:
  data_import.start_date, data_import.end_date, data_import.records = (
  save_temp_data_to_permanent(data_import)
  )
+ update_variables_for_station(data_import.station.station_code)
  data_import.status = "C"
  data_import.log = "Data ingestion completed successfully"
  getLogger("huey").info("Data ingestion for %s completed", data_import)
@@ -35,3 +38,12 @@ def ingest_data(data_import_pk: int) -> None:
  getLogger("huey").exception("Error ingesting data for %s", data_import)
  finally:
  data_import.save()
+ clear_cache()
+
+
+@task()
+def clear_cache() -> None:
+ """Clear the cache."""
+ from measurement.reporting import get_report_data_from_db
+
+ get_report_data_from_db.cache_clear()

measurement/dash_apps/data_report.py

@@ -103,26 +103,36 @@
 @app.callback(
  Output("data_report_graph", "figure"),
  [
- Input("temporality_drop", "value"),
- Input("station_drop", "value"),
- Input("variable_drop", "value"),
- Input("date_range_picker", "start_date"),
- Input("date_range_picker", "end_date"),
  Input("data_report_graph", "relayoutData"),
+ Input("display_button", "n_clicks"),
+ ],
+ [
+ State("temporality_drop", "value"),
+ State("station_drop", "value"),
+ State("variable_drop", "value"),
+ State("date_range_picker", "start_date"),
+ State("date_range_picker", "end_date"),
+ State("data_report_graph", "figure"),
  ],
 )
 def update_graph(
+ relayout_data: dict,
+ n_clicks: int,
  temporality: str,
  station: str,
  variable: str,
  start_time: str,
  end_time: str,
- relayout_data: dict,
+ figure: go.Figure,
  callback_context,
 ) -> go.Figure:
  ctx = callback_context
  triggered_id = ctx.triggered[0]["prop_id"].split(".")[0] if ctx.triggered else ""
 
+ if not n_clicks:
+ # After the first load, n_clicks is always > 0, so zooming works
+ return figure
+
  # This is cached, so it's not a big deal to call it multiple times
  data = get_report_data_from_db(
  station=station,
@@ -148,6 +158,11 @@ def update_graph(
  variable_name=Variable.objects.get(variable_code=variable).name,
  station_code=station,
  )
+ if "xaxis.range[0]" in relayout_data:
+ plot["layout"]["xaxis"]["range"] = [
+ relayout_data["xaxis.range[0]"],
+ relayout_data["xaxis.range[1]"],
+ ]
  return plot
 
  except Exception as e:
@@ -212,17 +227,37 @@ def download_csv_report(
  Output("data_alert_div", "children"),
  Output("csv_div", "children"),
  ],
- Input("data_report_graph", "figure"),
+ [
+ Input("temporality_drop", "value"),
+ Input("station_drop", "value"),
+ Input("variable_drop", "value"),
+ Input("date_range_picker", "start_date"),
+ Input("date_range_picker", "end_date"),
+ ],
+ State("data_report_graph", "figure"),
 )
-def update_alert(figure):
+def update_alert(
+ temporality: str,
+ station: str,
+ variable: str,
+ start_time: str,
+ end_time: str,
+ figure: go.Figure,
+):
  if figure["layout"]["title"]["text"] == "Data not found":
  alert = dbc.Alert(
  "No data was found with the selected criteria", color="warning"
  )
  return [alert], []
  else:
- button = html.Button("Download CSV", id="csv_button")
- return [], [button]
+ download = html.Button("Download CSV", id="csv_button")
+ display = html.Button(
+ "Display data",
+ id="display_button",
+ style={"margin-left": "10px"},
+ n_clicks=0,
+ )
+ return [], [download, display]
 
 
 @app.callback(

measurement/filters.py

@@ -1,8 +1,10 @@
 from decimal import Decimal
 
-from django.db.models import Max, Min
+from django.db.models import Max, Min, Q
 
+from station.models import Station
 from utilities.timezones import to_local_time
+from variable.models import Variable
 
 from .models import Measurement
 
@@ -19,14 +21,13 @@ def get_station_options(
  Returns:
  tuple[list[dict], str]: Options for the station dropdown, default value
  """
- stations_with_measurements = Measurement.objects.values_list(
- "station__station_code", flat=True
- ).distinct()
+ stations_with_measurements = Station.objects.filter(
+ ~Q(variables=""), station_code__in=station_codes
+ ).values_list("station_code", flat=True)
 
  station_options = [
  {"label": station_code, "value": station_code}
- for station_code in station_codes
- if station_code in stations_with_measurements
+ for station_code in stations_with_measurements
  ]
  station_value = station_options[0]["value"] if station_options else None
  return station_options, station_value
@@ -41,19 +42,19 @@ def get_variable_options(station: str) -> tuple[list[dict], str]:
  Returns:
  tuple[list[dict], str]: Options for the variable dropdown, default value
  """
- variable_dicts = (
- Measurement.objects.filter(station__station_code=station)
- .values("variable__name", "variable__variable_code")
- .distinct()
+ variable_codes = Station.objects.get(station_code=station).variables_list
+ variable_dicts = Variable.objects.filter(variable_code__in=variable_codes).values(
+ "name", "variable_code"
  )
 
  variable_options = [
  {
- "label": variable["variable__name"],
- "value": variable["variable__variable_code"],
+ "label": variable["name"],
+ "value": variable["variable_code"],
  }
  for variable in variable_dicts
  ]
+
  variable_value = variable_options[0]["value"] if variable_options else None
  return variable_options, variable_value
 

measurement/reporting.py

@@ -173,7 +173,7 @@ def save_report_data(data: pd.DataFrame) -> None:
  )
 
 
-@lru_cache
+@lru_cache(1)
 def get_report_data_from_db(
  station: str,
  variable: str,

station/functions.py

@@ -0,0 +1,38 @@
+from measurement.models import Measurement
+
+from .models import Station
+
+
+def update_variables_for_station(*station_codes) -> None:
+ """Update the variables for the given station codes.
+
+ The variables are updated based on the measurements associated with the station.
+ The variables are saved as a comma-separated string in the variables field of the
+ station model.
+
+ Args:
+ station_codes (tuple[str]): Station codes for which to update the variables.
+ If not provided, all station codes with measurements are considered.
+ """
+
+ # We get the station codes from the Measurement model if not provided
+ # Only station codes with measurements are considered
+ station_codes = (
+ station_codes
+ or Measurement.objects.values_list(
+ "station__station_code", flat=True
+ ).distinct()
+ )
+
+ # Get the variables for each station and save them as a comma-separated string
+ for station_code in station_codes:
+ variables = (
+ Measurement.objects.filter(station__station_code=station_code)
+ .values_list("variable__variable_code", flat=True)
+ .distinct()
+ )
+ if variables:
+ station = Station.objects.get(station_code=station_code)
+ station.variables = variables = ",".join(variables)
+ station.full_clean()
+ station.save()

station/migrations/0018_station_variables.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.2 on 2024-10-18 10:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('station', '0017_alter_basin_owner_alter_country_owner_and_more'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='station',
+ name='variables',
+ field=models.CharField(default='', editable=False, help_text='Comma-separated list of variables measured by the station.', max_length=100, verbose_name='Stored variables'),
+ ),
+ ]

station/models.py

@@ -306,6 +306,7 @@ class Station(PermissionsBase):
  influence_km (Decimal): Area of influence in km2.
  station_file (ImageField): Photography of the station.
  station_external (bool): Is the station external?
+ variables (str): Comma-separated list of variables measured by the station.
  """
 
  VISIBILITY_LEVELS = [
@@ -433,6 +434,26 @@ class Station(PermissionsBase):
  station_external = models.BooleanField(
  "External", default=False, help_text="Is the station external?"
  )
+ variables = models.CharField(
+ "Stored variables",
+ max_length=100,
+ null=False,
+ blank=False,
+ default="",
+ editable=False,
+ help_text="Comma-separated list of variables measured by the station.",
+ )
+
+ @property
+ def variables_list(self) -> list[str]:
+ """Return the list of variables measured by the station.
+
+ Only variables with data in the database are returned.
+
+ Returns:
+ list[str]: List of variables measured by the station.
+ """
+ return self.variables.split(",") if self.variables else []
 
  def __str__(self) -> str:
  """Return the station code."""

tests/measurement/test_validation.py

@@ -48,6 +48,8 @@ def setUp(self):
  Measurement.objects.bulk_create(records)
 
  def test_get_data_to_validate(self):
+ from datetime import timedelta
+
  from measurement.models import Measurement
  from measurement.validation import get_data_to_validate
 
@@ -61,8 +63,8 @@ def test_get_data_to_validate(self):
  data = get_data_to_validate(
  station=self.station.station_code,
  variable=self.variable.variable_code,
- start_time=self.start.strftime("%Y-%m-%d"),
- end_time=self.end.strftime("%Y-%m-%d"),
+ start_time=(self.start - timedelta(days=1)).strftime("%Y-%m-%d"),
+ end_time=(self.end + timedelta(days=1)).strftime("%Y-%m-%d"),
  is_validated=False,
  )
  self.assertEqual(len(data), len(self.date_range) - 1)

utilities/load_initial_data.py

@@ -7,6 +7,7 @@
 from django.core.management import execute_from_command_line
 
 from measurement.models import Measurement
+from station.functions import update_variables_for_station
 
 # The order matters so that foreignkey objects exist when an entry is created.
 # The files are named app_model
@@ -52,3 +53,5 @@
 for meas in Measurement.objects.all():
  meas.full_clean()
  meas.save()
+
+update_variables_for_station()