Debugging became easier (less IO and Disk Space)

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog
 
+## 2.0.14
+
+**Debugging became easier (less IO and Disk Space)**
+- Removed `Store Debug Data` switch (Moved to the API endpoints below)
+- Removed WebSocket messages sensors (Moved to the API endpoints below)
+- Add endpoints to expose the data was previously stored to files and the messages counters
+
+| Endpoint Name              | Method | Description                                                                                         |
+|----------------------------|--------|-----------------------------------------------------------------------------------------------------|
+| /api/edgeos/list           | GET    | List all the endpoints available (supporting multiple integrations), available once for integration |
+| /api/edgeos/{ENTRY_ID}/ha  | GET    | JSON of all HA processed data before sent to entities including messages counters, per integration  |
+| /api/edgeos/{ENTRY_ID}/api | GET    | JSON of all raw data from the EdgeOS API, per integration                                           |
+| /api/edgeos/{ENTRY_ID}/ws  | GET    | JSON of all raw data from the EdgeOS WebSocket, per integration                                     |
+
+**Authentication: Requires long-living token from HA**
+
 ## 2.0.13
 
 - Add support for all interfaces but `loopback` [#76](https://github.com/elad-bar/ha-edgeos/issues/76)

README.md

@@ -85,12 +85,8 @@ logger:
 | {Router Name} RAM                   | Sensor        | Represents RAM usage                                                      |                                               |
 | {Router Name} Uptime                | Sensor        | Represents last time the EdgeOS was restarted                             |                                               |
 | {Router Name} Unknown devices       | Sensor        | Represents number of devices leased by the DHCP server                    | Attributes holds the leased hostname and IPs  |
-| {Router Name} Received Messages     | Sensor        | Represents the number of WS messages handled                              |                                               |
-| {Router Name} Ignored Messages      | Sensor        | Represents the number of WS messages ignored                              |                                               |
-| {Router Name} Error Messages        | Sensor        | Represents the number of WS messages of errors                            |                                               |
 | {Router Name} Firmware Updates      | Binary Sensor | New firmware available indication                                         | Attributes holds the url and new release name |
 | {Router Name} Log incoming messages | Switch        | Sets whether to log WebSocket incoming messages for debugging             |                                               |
-| {Router Name} Store Debug Data      | Switch        | Sets whether to store API and WebSocket latest data for debugging         |                                               |
 
 *Changing the unit will reload the integration*
 
@@ -150,3 +146,14 @@ data:
 ```
 
 *Changing the unit will reload the integration*
+
+## Endpoints
+
+| Endpoint Name              | Method | Description                                                                                         |
+|----------------------------|--------|-----------------------------------------------------------------------------------------------------|
+| /api/edgeos/list           | GET    | List all the endpoints available (supporting multiple integrations), available once for integration |
+| /api/edgeos/{ENTRY_ID}/ha  | GET    | JSON of all HA processed data before sent to entities, per integration                              |
+| /api/edgeos/{ENTRY_ID}/api | GET    | JSON of all raw data from the EdgeOS API, per integration                                           |
+| /api/edgeos/{ENTRY_ID}/ws  | GET    | JSON of all raw data from the EdgeOS WebSocket, per integration                                     |
+
+**Authentication: Requires long-living token from HA**

custom_components/edgeos/component/api/storage_api.py

@@ -1,9 +1,8 @@
 """Storage handlers."""
 from __future__ import annotations
 
-from datetime import datetime
-import json
 import logging
+import sys
 from typing import Awaitable, Callable
 
 from homeassistant.core import HomeAssistant
@@ -14,12 +13,16 @@
 from ...core.api.base_api import BaseAPI
 from ...core.helpers.enums import ConnectivityStatus
 from ..helpers.const import *
+from ..models.base_view import EdgeOSBaseView
 
 _LOGGER = logging.getLogger(__name__)
 
 
 class StorageAPI(BaseAPI):
-    _storage: Store
+    _stores: dict[str, Store] | None
+    _views: dict[str, EdgeOSBaseView] | None
+    _config_data: ConfigData | None
+    _data: dict
 
     def __init__(self,
                  hass: HomeAssistant,
@@ -29,29 +32,14 @@ def __init__(self,
 
         super().__init__(hass, async_on_data_changed, async_on_status_changed)
 
-        self._storages = None
+        self._config_data = None
+        self._stores = None
+        self._views = None
+        self._data = {}
 
     @property
     def _storage_config(self) -> Store:
-        storage = self._storages.get(STORAGE_DATA_FILE_CONFIG)
-
-        return storage
-
-    @property
-    def _storage_api(self) -> Store:
-        storage = self._storages.get(STORAGE_DATA_FILE_API_DEBUG)
-
-        return storage
-
-    @property
-    def _storage_ws(self) -> Store:
-        storage = self._storages.get(STORAGE_DATA_FILE_WS_DEBUG)
-
-        return storage
-
-    @property
-    def _storage_ha(self) -> Store:
-        storage = self._storages.get(STORAGE_DATA_FILE_HA_DEBUG)
+        storage = self._stores.get(STORAGE_DATA_FILE_CONFIG)
 
         return storage
 
@@ -104,17 +92,52 @@ def update_api_interval(self):
         return result
 
     async def initialize(self, config_data: ConfigData):
-        storages = {}
-        entry_id = config_data.entry.entry_id
+        self._config_data = config_data
+
+        self._initialize_routes()
+        self._initialize_storages()
+
+        await self._async_load_configuration()
+
+    def _initialize_storages(self):
+        stores = {}
+
+        entry_id = self._config_data.entry.entry_id
 
         for storage_data_file in STORAGE_DATA_FILES:
             file_name = f"{DOMAIN}.{entry_id}.{storage_data_file}.json"
 
-            storages[storage_data_file] = Store(self.hass, STORAGE_VERSION, file_name, encoder=JSONEncoder)
+            stores[storage_data_file] = Store(self.hass, STORAGE_VERSION, file_name, encoder=JSONEncoder)
 
-        self._storages = storages
+        self._stores = stores
 
-        await self._async_load_configuration()
+    def _initialize_routes(self):
+        try:
+            main_view_data = {}
+            entry_id = self._config_data.entry.entry_id
+
+            for key in STORAGE_API_DATA:
+                view = EdgeOSBaseView(self.hass, key, self._get_data, entry_id)
+
+                main_view_data[key] = view.url
+
+                self.hass.http.register_view(view)
+
+            main_view = self.hass.data.get(MAIN_VIEW)
+
+            if main_view is None:
+                main_view = EdgeOSBaseView(self.hass, STORAGE_API_LIST, self._get_data)
+
+                self.hass.http.register_view(main_view)
+                self.hass.data[MAIN_VIEW] = main_view
+
+            self._data[STORAGE_API_LIST] = main_view_data
+
+        except Exception as ex:
+            exc_type, exc_obj, tb = sys.exc_info()
+            line_number = tb.tb_lineno
+
+            _LOGGER.error(f"Failed to async_component_initialize, error: {ex}, line: {line_number}")
 
     async def _async_load_configuration(self):
         """Load the retained data from store and return de-serialized data."""
@@ -175,13 +198,6 @@ async def set_log_incoming_messages(self, enabled: bool):
 
         await self._async_save()
 
-    async def set_store_debug_data(self, enabled: bool):
-        _LOGGER.debug(f"Set store debug data to {enabled}")
-
-        self.data[STORAGE_DATA_STORE_DEBUG_DATA] = enabled
-
-        await self._async_save()
-
     async def set_consider_away_interval(self, interval: int):
         _LOGGER.debug(f"Changing {STORAGE_DATA_CONSIDER_AWAY_INTERVAL}: {interval}")
 
@@ -204,43 +220,51 @@ async def set_update_api_interval(self, interval: int):
         await self._async_save()
 
     async def debug_log_api(self, data: dict):
-        if self.store_debug_data and data is not None:
-            await self._storage_api.async_save(self._get_json_data(data))
+        self._data[STORAGE_API_DATA_API] = data
 
     async def debug_log_ws(self, data: dict):
-        if self.store_debug_data and data is not None:
-            await self._storage_ws.async_save(self._get_json_data(data))
+        self._data[STORAGE_API_DATA_WS] = data
 
     async def debug_log_ha(self, data: dict):
-        if self.store_debug_data and data is not None:
-            clean_data = {}
-            for key in data:
-                if key in [DEVICE_LIST, API_DATA_INTERFACES]:
-                    new_item = {}
-                    items = data.get(key, {})
+        clean_data = {}
+        for key in data:
+            if key in [DEVICE_LIST, API_DATA_INTERFACES]:
+                new_item = {}
+                items = data.get(key, {})
 
-                    for item_key in items:
-                        item = items.get(item_key)
-                        new_item[item_key] = item.to_dict()
+                for item_key in items:
+                    item = items.get(item_key)
+                    new_item[item_key] = item.to_dict()
 
-                    clean_data[key] = new_item
+                clean_data[key] = new_item
 
-                elif key in [API_DATA_SYSTEM]:
-                    item = data.get(key)
-                    clean_data[key] = item.to_dict()
+            elif key in [API_DATA_SYSTEM]:
+                item = data.get(key)
+                clean_data[key] = item.to_dict()
 
-            await self._storage_ha.async_save(self._get_json_data(clean_data))
+            else:
+                clean_data[key] = data.get(key)
 
-    def _get_json_data(self, data: dict):
-        json_data = json.dumps(data, default=self.json_converter, sort_keys=True, indent=4)
+        self._data[STORAGE_API_DATA_HA] = clean_data
 
-        result = json.loads(json_data)
+    def _get_data(self, key):
+        is_list = key == STORAGE_API_LIST
 
-        return result
+        data = {} if is_list else self._data.get(key)
+
+        if is_list:
+            raw_data = self._data.get(key)
+            current_entry_id = self._config_data.entry.entry_id
+
+            for entry_id in self.hass.data[DATA].keys():
+                entry_data = {}
+
+                for raw_data_key in raw_data:
+                    url_raw = raw_data.get(raw_data_key)
+                    url = url_raw.replace(current_entry_id, entry_id)
+
+                    entry_data[raw_data_key] = url
+
+                data[entry_id] = entry_data
 
-    @staticmethod
-    def json_converter(data):
-        if isinstance(data, datetime):
-            return data.__str__()
-        if isinstance(data, dict):
-            return data.__dict__
+        return data

custom_components/edgeos/component/helpers/const.py

@@ -27,17 +27,23 @@
 DEFAULT_CONSIDER_AWAY_INTERVAL = timedelta(minutes=3)
 
 STORAGE_DATA_FILE_CONFIG = "config"
-STORAGE_DATA_FILE_API_DEBUG = "debug.api"
-STORAGE_DATA_FILE_WS_DEBUG = "debug.ws"
-STORAGE_DATA_FILE_HA_DEBUG = "debug.ha"
+STORAGE_API_LIST = "list"
+STORAGE_API_DATA_API = "api"
+STORAGE_API_DATA_WS = "ws"
+STORAGE_API_DATA_HA = "ha"
 
 STORAGE_DATA_FILES = [
-    STORAGE_DATA_FILE_CONFIG,
-    STORAGE_DATA_FILE_API_DEBUG,
-    STORAGE_DATA_FILE_WS_DEBUG,
-    STORAGE_DATA_FILE_HA_DEBUG
+    STORAGE_DATA_FILE_CONFIG
 ]
 
+STORAGE_API_DATA = [
+    STORAGE_API_DATA_API,
+    STORAGE_API_DATA_WS,
+    STORAGE_API_DATA_HA
+]
+
+MESSAGES_COUNTER_SECTION = "messages"
+
 STORAGE_DATA_MONITORED_INTERFACES = "monitored-interfaces"
 STORAGE_DATA_MONITORED_DEVICES = "monitored-devices"
 STORAGE_DATA_UNIT = "unit"
@@ -106,6 +112,12 @@
 WS_IGNORED_MESSAGES = "ignored-messages"
 WS_ERROR_MESSAGES = "error-messages"
 
+WS_MESSAGES = [
+    WS_RECEIVED_MESSAGES,
+    WS_IGNORED_MESSAGES,
+    WS_ERROR_MESSAGES
+]
+
 UPDATE_DATE_ENDPOINTS = [
     API_DATA_SYS_INFO,
     API_DATA_DHCP_STATS,