modules.hpi.twitter_android: initial experimental module via object n…

…ormaliser using HPI

also see #28

setup.py

@@ -50,6 +50,12 @@ def main() -> None:
                 'mypy', 'lxml',  # lxml for mypy coverage report
             ],
             'zstd'   : ['kompress[zstd]'],
+            'HPI': [  # for bleanser.modules.hpi
+                # 'HPI', # pypi version
+                'HPI @ git+https://github.com/karlicoss/hpi.git',   # uncomment to test against github version (useful for one-off CI run)
+                # 'HPI @ git+file:///DUMMY/path/to/local/hpi@branch', # uncomment to test against version on the disc
+                # note: sometimes you need to use file://DUMMY?? wtf?..
+            ],
         },
 
 

src/bleanser/modules/hpi/twitter_android.py

@@ -0,0 +1,41 @@
+# NOTE: this is experimental for now, best to use the corresponding module bleanser.modules.* instead
+import os
+from pathlib import Path
+from typing import Any, Iterator
+
+from bleanser.core.modules.extract import ExtractObjectsNormaliser
+
+from my.core.cfg import tmp_config
+
+## disable cache, otherwise it's gonna flush it all the time
+# TODO this should be in some sort of common module
+os.environ['CACHEW_DISABLE'] = '*'
+os.environ.pop('ENLIGHTEN_ENABLE', None)
+os.environ['LOGGING_LEVEL_my_twitter_android'] = 'WARNING'
+##
+
+import my.twitter.android as twitter_android
+
+
+class Normaliser(ExtractObjectsNormaliser):
+    MULTIWAY = True
+    PRUNE_DOMINATED = True
+
+    def extract_objects(self, path: Path) -> Iterator[Any]:
+        class config:
+            class twitter:
+                class android:
+                    export_path = path
+
+        with tmp_config(modules=twitter_android.__name__, config=config):
+            assert len(twitter_android.inputs()) == 1  # sanity check to make sure tmp_config worked as expected
+            for x in twitter_android.bookmarks():
+                yield 'bookmark', x
+            for x in twitter_android.likes():
+                yield 'like', x
+            for x in twitter_android.tweets():
+                yield 'tweet', x
+
+
+if __name__ == '__main__':
+    Normaliser.main()

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 minversion = 3.21
 # relies on the correct version of Python installed
-envlist = ruff,tests,mypy
+envlist = ruff,tests,mypy,mypy-hpi
 # https://github.com/tox-dev/tox/issues/20#issuecomment-247788333
 # hack to prevent .tox from crapping to the project directory
 toxworkdir = {env:TOXWORKDIR_BASE:}{toxinidir}/.tox
@@ -34,16 +34,28 @@ commands =
     {envpython} -m pip install --use-pep517 -e .[testing]
     # posargs allow test filtering, e.g. tox ... -- -k test_name
     {envpython} -m pytest \
-        --pyargs {[testenv]package_name} \
+        --pyargs {[testenv]package_name} --ignore-glob 'src/bleanser/modules/hpi/*' \
         {posargs}
 
 
 [testenv:mypy]
 commands =
     {envpython} -m pip install --use-pep517 -e .[testing]
     {envpython} -m mypy --install-types --non-interactive \
-        -p {[testenv]package_name}       \
+        # note: hpi modules are tested below
+        -p {[testenv]package_name} --exclude 'hpi/*' \
         # txt report is a bit more convenient to view on CI
         --txt-report  .coverage.mypy     \
         --html-report .coverage.mypy     \
         {posargs}
+
+
+[testenv:mypy-hpi]
+commands =
+    {envpython} -m pip install --use-pep517 -e .[testing,HPI]
+    {envpython} -m mypy --install-types --non-interactive \
+        -p {[testenv]package_name}.modules.hpi \
+        # txt report is a bit more convenient to view on CI
+        --txt-report  .coverage.mypy-hpi \
+        --html-report .coverage.mypy-hpi \
+        {posargs}