Merge pull request #57 from SunbirdAI/new_device_recordings

Add endpoints for new device metrics and sound inferences

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+ ignore = D203, E402, F403, F405, W503, W605
+ exclude = .git,env,__pycache__,docs/source/conf.py,old,build,dist, *migrations*,env,settings.py,local_settings.py,example.local_settings.py
+ max-complexity = 10
+ max-line-length = 119

.gitattributes

@@ -0,0 +1 @@
+* text=auto

.gitignore

@@ -1,10 +1,12 @@
 myenv/
 venv/
+env/
 __pycache__
 .coverage
 .idea
 .vscode
 .env
+.env.example
 *.csv
 *.ipynb
 data

.isort.cfg

@@ -0,0 +1,7 @@
+[settings]
+multi_line_output=3
+include_trailing_comma=True
+force_grid_wrap=0
+use_parentheses=True
+line_length=88
+skip=env,migrations,local_settings.py

Dockerfile

@@ -13,7 +13,7 @@ WORKDIR /code
 # RUN apt-get install -y cron
 
 # Upgrade Pip
-RUN pip install --upgrade pip
+# RUN pip install --upgrade pip
 
 # Install dependencies
 COPY requirements.txt /code/

analysis/aggregate_data.py

@@ -1,9 +1,11 @@
 import pandas as pd
-from .models import DailyAggregate, HourlyAggregate
+
 from devices.models import Device, location_category_information
 
-class Aggregate:
+from .models import DailyAggregate, HourlyAggregate
 
+
+class Aggregate:
  def __init__(self, data):
  self.data = data
  self.locations = self.get_device_locations()
@@ -29,39 +31,50 @@ def aggregate_hourly(self, time_uploaded=None):
  grouped_data = df.groupby("device_id", sort=False)
  for device_name, device_df in grouped_data:
  if 6 <= hour < 22:
- threshold = location_category_information[self.locations[device_name].category]["day_limit"]
+ threshold = location_category_information[
+ self.locations[device_name].category
+ ]["day_limit"]
  else:
- threshold = location_category_information[self.locations[device_name].category]["night_limit"]
+ threshold = location_category_information[
+ self.locations[device_name].category
+ ]["night_limit"]
 
  HourlyAggregate.objects.create(
- device = Device.objects.get(device_id=device_name),
+ device=Device.objects.get(device_id=device_name),
  date=time_uploaded,
- hour = hour,
- hourly_avg_db_level = device_df["db_level"].mean(),
- hourly_median_db_level = device_df["db_level"].median(),
- hourly_max_db_level = device_df["db_level"].max(),
- hourly_no_of_exceedances = len(device_df[device_df["db_level"] > threshold])
+ hour=hour,
+ hourly_avg_db_level=device_df["db_level"].mean(),
+ hourly_median_db_level=device_df["db_level"].median(),
+ hourly_max_db_level=device_df["db_level"].max(),
+ hourly_no_of_exceedances=len(
+ device_df[device_df["db_level"] > threshold]
+ ),
  )
 
  return None
 
-
  def aggregate_daily(self, time_period):
  df = self.prepare_data()
  grouped_data = df.groupby("device_id", sort=False)
  for device_name, device_df in grouped_data:
  if time_period == "daytime":
- threshold = location_category_information[self.locations[device_name].category]["day_limit"]
+ threshold = location_category_information[
+ self.locations[device_name].category
+ ]["day_limit"]
  else:
- threshold = location_category_information[self.locations[device_name].category]["night_limit"]
+ threshold = location_category_information[
+ self.locations[device_name].category
+ ]["night_limit"]
 
  DailyAggregate.objects.create(
- device = Device.objects.get(device_id=device_name),
- time_period = time_period,
- daily_avg_db_level = device_df["db_level"].mean(),
- daily_median_db_level = device_df["db_level"].median(),
- daily_max_db_level = device_df["db_level"].max(),
- daily_no_of_exceedances = len(device_df[device_df["db_level"] > threshold])
+ device=Device.objects.get(device_id=device_name),
+ time_period=time_period,
+ daily_avg_db_level=device_df["db_level"].mean(),
+ daily_median_db_level=device_df["db_level"].median(),
+ daily_max_db_level=device_df["db_level"].max(),
+ daily_no_of_exceedances=len(
+ device_df[device_df["db_level"] > threshold]
+ ),
  )
 
  return None

analysis/apps.py

@@ -2,5 +2,5 @@
 
 
 class AnalysisConfig(AppConfig):
- default_auto_field = 'django.db.models.BigAutoField'
- name = 'analysis'
+ default_auto_field = "django.db.models.BigAutoField"
+ name = "analysis"

analysis/cron.py

@@ -1,18 +1,17 @@
 import logging
-import pytz
 from datetime import datetime
 
+import pytz
 from django.db.models import Avg, Max, Sum
 
-from noise_dashboard.settings import TIME_ZONE
-from .models import DailyAnalysis
-from devices.models import Device
 from device_metrics.models import DeviceMetrics
+from devices.models import Device
+from noise_dashboard.settings import TIME_ZONE
 
+from .models import DailyAnalysis
 
-logging.basicConfig(filename="app.log", 
- format="%(asctime)s - %(message)s",
- level=logging.INFO
+logging.basicConfig(
+ filename="app.log", format="%(asctime)s - %(message)s", level=logging.INFO
 )
 
 
@@ -27,19 +26,19 @@ def aggregate_daily_metrics():
  device=device.id,
  time_uploaded__year=today.year,
  time_uploaded__month=today.month,
- time_uploaded__day=today.day
+ time_uploaded__day=today.day,
  )
 
- daily_avg_db_level = device_metrics.aggregate(Avg('avg_db_level'))
- daily_max_db_level = device_metrics.aggregate(Max('max_db_level'))
- daily_no_of_exceedances = device_metrics.aggregate(Sum('no_of_exceedances'))
+ daily_avg_db_level = device_metrics.aggregate(Avg("avg_db_level"))
+ daily_max_db_level = device_metrics.aggregate(Max("max_db_level"))
+ daily_no_of_exceedances = device_metrics.aggregate(Sum("no_of_exceedances"))
 
  DailyAnalysis.objects.create(
- date_analyzed = today,
- daily_avg_db_level=daily_avg_db_level['avg_db_level__avg'],
- daily_max_db_level=daily_max_db_level['max_db_level__max'],
- daily_no_of_exceedances=daily_no_of_exceedances['no_of_exceedances__sum'],
- device=device
+ date_analyzed=today,
+ daily_avg_db_level=daily_avg_db_level["avg_db_level__avg"],
+ daily_max_db_level=daily_max_db_level["max_db_level__max"],
+ daily_no_of_exceedances=daily_no_of_exceedances["no_of_exceedances__sum"],
+ device=device,
  )
 
- logging.info(f'Daily device metrics for device {device} aggregated')
+ logging.info(f"Daily device metrics for device {device} aggregated")

analysis/models.py

@@ -1,17 +1,18 @@
 import uuid
+
+from django.core.files.storage import get_storage_class
 from django.db import models
 from django.utils import timezone
-from django.core.files.storage import get_storage_class
 
 from devices.models import Device
 
-
 media_storage = get_storage_class()()
 
 
 def recording_directory(instance, filename):
  time_uploaded = instance.time_uploaded.strftime("%Y-%m-%dT%H:%M:%S")
- return f'metrics/{instance.device.device_id}/{instance.device.device_id}-{time_uploaded}-{filename}'
+ return f"metrics/{instance.device.device_id}/{instance.device.device_id}-{time_uploaded}-{filename}"
+
 
 class MetricsTextFile(models.Model):
  time_uploaded = models.DateTimeField(auto_now_add=True)
@@ -28,11 +29,7 @@ def filename(self):
 
 
 class HourlyAggregate(models.Model):
- id = models.UUIDField(
- primary_key=True,
- default=uuid.uuid4,
- editable=False
- )
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
  date = models.DateTimeField(default=timezone.now)
  hour = models.PositiveIntegerField(null=True)
  hourly_avg_db_level = models.FloatField(null=True)
@@ -43,16 +40,11 @@ class HourlyAggregate(models.Model):
 
 
 class DailyAggregate(models.Model):
-
  class TimePeriod(models.TextChoices):
- DAYTIME = 'daytime'
- NIGHTTIME = 'nighttime'
-
- id = models.UUIDField(
- primary_key=True,
- default=uuid.uuid4,
- editable=False
- )
+ DAYTIME = "daytime"
+ NIGHTTIME = "nighttime"
+
+ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
  date = models.DateTimeField(default=timezone.now)
  time_period = models.CharField(max_length=10, choices=TimePeriod.choices)
  daily_avg_db_level = models.FloatField(null=True)

analysis/parse_file_metrics.py

@@ -4,7 +4,7 @@ def parse_file(file, device_id):
  """
  metrics = []
  missing_values = 0
- with file.open('r') as f:
+ with file.open("r") as f:
  lines = f.readlines()
  for line in lines:
  line.strip()
@@ -14,7 +14,9 @@ def parse_file(file, device_id):
  try:
  decibel = float(decibel_string.split(":")[1])
  time_uploaded = time_uploaded_string[3:]
- metrics.append({'device_id': device_id, 'db_level': decibel, 'date': time_uploaded})
+ metrics.append(
+ {"device_id": device_id, "db_level": decibel, "date": time_uploaded}
+ )
  except ValueError as e:
  missing_values += 1
  print(f"Error: {e}")

analysis/serializers.py

@@ -1,44 +1,43 @@
 from rest_framework import serializers
-from devices.models import Device
+
 import devices.serializers
-from .models import HourlyAggregate, DailyAggregate, MetricsTextFile
+from devices.models import Device
+
+from .models import DailyAggregate, HourlyAggregate, MetricsTextFile
+
 
 class UploadMetricsTextFileSerializer(serializers.ModelSerializer):
- device = serializers.SlugRelatedField(queryset=Device.objects.all(), slug_field='device_id')
+ device = serializers.SlugRelatedField(
+ queryset=Device.objects.all(), slug_field="device_id"
+ )
 
  class Meta:
  model = MetricsTextFile
- fields = [
- 'id', 'metrics_file', 'device'
- ]
- read_only_fields = ['time_uploaded']
+ fields = ["id", "metrics_file", "device"]
+ read_only_fields = ["time_uploaded"]
 
  def to_representation(self, instance):
- self.fields['device'] = devices.serializers.DeviceSerializer(read_only=True)
- return {
- "result": "success"
- }
+ self.fields["device"] = devices.serializers.DeviceSerializer(read_only=True)
+ return {"result": "success"}
 
 
 class ListMetricsTextFileSerializer(serializers.ModelSerializer):
- device = serializers.SlugRelatedField(queryset=Device.objects.all(), slug_field='device_id')
+ device = serializers.SlugRelatedField(
+ queryset=Device.objects.all(), slug_field="device_id"
+ )
 
  class Meta:
  model = MetricsTextFile
- fields = [
- 'id', 'time_uploaded', 'device', 'text_file_url'
- ]
+ fields = ["id", "time_uploaded", "device", "text_file_url"]
 
 
 class HourlyAnalysisSerializer(serializers.ModelSerializer):
-
  class Meta:
  model = HourlyAggregate
  fields = "__all__"
 
 
 class DailyAnalysisSerializer(serializers.ModelSerializer):
-
  class Meta:
  model = DailyAggregate
  fields = "__all__"

analysis/urls.py

@@ -1,25 +1,32 @@
-from django.urls import path, include
-from .views import (
+from django.urls import include, path
+from rest_framework.routers import SimpleRouter
+
+from .views import ( # DailyAnalysisView,; ReceiveIoTDataView,
+ AggregateMetricsView,
  HourlyAnalysisView,
- # DailyAnalysisView,
- # ReceiveIoTDataView,
+ ListMetricsFilesView,
  ReceiveMetricsFileViewSet,
- AggregateMetricsView,
- ListMetricsFilesView
 )
-from rest_framework.routers import SimpleRouter
 
 router = SimpleRouter()
-router.register('metrics-file', ReceiveMetricsFileViewSet)
+router.register("metrics-file", ReceiveMetricsFileViewSet)
 
 # urlpatterns = [
 # path('hourly/', HourlyAnalysisView.as_view(), name='hourly_analysis'),
 # path('daily/', DailyAnalysisView.as_view(), name='daily_analysis'),
 # path('receive_iot_data/', ReceiveIoTDataView.as_view(), name='iot_data'),
 # ]
 urlpatterns = [
- path('aggregate-metrics/', AggregateMetricsView.as_view(), name='aggregate_metrics'),
- path('list-metrics/<str:device_id>', ListMetricsFilesView.as_view(), name='list_metric_files'),
- path('hourly/<str:device_id>', HourlyAnalysisView.as_view(), name='hourly_analysis'),
- path('', include(router.urls))
+ path(
+ "aggregate-metrics/", AggregateMetricsView.as_view(), name="aggregate_metrics"
+ ),
+ path(
+ "list-metrics/<str:device_id>",
+ ListMetricsFilesView.as_view(),
+ name="list_metric_files",
+ ),
+ path(
+ "hourly/<str:device_id>", HourlyAnalysisView.as_view(), name="hourly_analysis"
+ ),
+ path("", include(router.urls)),
 ]