feat: add to_dict() methods and remove e2e tests

Taskfile.yml

@@ -11,7 +11,6 @@ tasks:
     cmds:
       - task unit
       - task lint-fix
-      - task e2e-smoke
 
   p:
     desc: Shortcut for `task pre-commit`.
@@ -23,7 +22,6 @@ tasks:
     cmds:
       - task unit
       - task lint
-      - task e2e
 
 ######################
 ### Run Unit Tests ###
@@ -64,18 +62,7 @@ tasks:
 ### Run End-to-end Tests ###
 ############################
 
-  e2e:
-    desc: Run repeated end-to-end tests.
-    cmds:
-      # The CLI_ARGS variable allows passing custom arguments to the end-to-end test after "--". 
-      # For instance, to run the test with specific parameters, use the command: 
-      # task e2e -- --tests-per-core=5 --cores=2 --limit_documents=2
-      - poetry run python -m tests.e2e.sec_parser {{.CLI_ARGS}}
 
-  e2e-smoke:
-    desc: Run a single end-to-end test for a single document. This is a useful way to verify if the parser is functioning (a so-called "smoke" test).
-    cmds:
-      - poetry run python -m tests.e2e.sec_parser --tests-per-core=1 --cores=1 --limit_documents=1
 
 #######################
 ### Developer Tools ###

pyproject.toml

@@ -30,6 +30,7 @@ lxml = "^4.9.3"
 httpx = "^0.24.1"
 cssutils = "^2.7.1"
 python-dotenv = "^1.0.0"
+xxhash = "^3.4.1"
 
 
 [tool.poetry.group.dev.dependencies]

sec_parser/processing_engine/html_tag.py

@@ -3,12 +3,16 @@
 import warnings
 
 import bs4
+import xxhash
+from frozendict import frozendict
 
 from sec_parser.exceptions import SecParserValueError
 from sec_parser.utils.bs4_.contains_tag import contains_tag
 from sec_parser.utils.bs4_.is_unary_tree import is_unary_tree
 from sec_parser.utils.bs4_.text_styles_metrics import compute_text_styles_metrics
 
+TEXT_PREVIEW_LENGTH = 40
+
 
 class HtmlTag:
     """
@@ -41,6 +45,37 @@ def __init__(
         self._is_unary_tree: bool | None = None
         self._first_deepest_tag: HtmlTag | None | NotSetType = NotSet
         self._text_styles_metrics: dict[tuple[str, str], float] | None = None
+        self._frozen_dict: frozendict | None = None
+        self._source_code: str | None = None
+
+    def get_source_code(self) -> str:
+        if self._source_code is None:
+            self._source_code = str(self._bs4)
+        return self._source_code
+
+    def _generate_preview(self, text: str) -> str:
+        """Generate a preview of the text with a specified length."""
+        text = text.replace("\n", " ").strip()
+        return (
+            text[: TEXT_PREVIEW_LENGTH // 2]
+            + f"...[{len(text) - TEXT_PREVIEW_LENGTH}]..."
+            + text[-TEXT_PREVIEW_LENGTH // 2 :]
+            if len(text) > TEXT_PREVIEW_LENGTH
+            else text
+        )
+
+    def to_dict(self) -> frozendict:
+        """Compute the hash of the HTML tag."""
+        if self._frozen_dict is None:
+            self._frozen_dict = frozendict(
+                {
+                    "tag_name": self._bs4.name,
+                    "text_preview": self._generate_preview(self.get_text()),
+                    "html_preview": self._generate_preview(self.get_source_code()),
+                    "html_hash": xxhash.xxh32(self.get_source_code()).hexdigest(),
+                },
+            )
+        return self._frozen_dict
 
     def get_text(self) -> str:
         """

sec_parser/semantic_elements/abstract_semantic_element.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import ABC
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from sec_parser.exceptions import SecParserValueError
 
@@ -37,8 +37,14 @@ def convert_from(
         """Convert the semantic element into another semantic element type."""
         return cls(source.html_tag)
 
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "cls_name": self.__class__.__name__,
+            **self.html_tag.to_dict(),
+        }
 
-class AbstractLevelElement(AbstractSemanticElement, ABC):
+
+class AbstractLevelElement(AbstractSemanticElement):
     """
     The AbstractLevelElement class provides a level attribute to semantic elements.
     It represents hierarchical levels in the document structure. For instance,
@@ -69,6 +75,12 @@ def convert_from(
     ) -> AbstractLevelElement:
         return cls(source.html_tag, level=level)
 
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            **super().to_dict(),
+            "level": self.level,
+        }
+
 
 class InvalidLevelError(SecParserValueError):
     pass

sec_parser/semantic_elements/composite_semantic_element.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from sec_parser.semantic_elements.abstract_semantic_element import (
     AbstractSemanticElement,
@@ -65,3 +65,9 @@ def convert_from(
             html_tag=source.html_tag,
             inner_elements=inner_elements,
         )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            **super().to_dict(),
+            "inner_elements": len(self.inner_elements),
+        }

sec_parser/semantic_elements/highlighted_text_element.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from sec_parser.semantic_elements.abstract_semantic_element import (
     AbstractSemanticElement,
@@ -42,6 +42,12 @@ def convert_from(
             style=style,
         )
 
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            **super().to_dict(),
+            "text_style": asdict(self.style),
+        }
+
 
 @dataclass(frozen=True)
 class TextStyle: