Unit tests for all Flowcb/Accept plugins (#757)

* Bunch more callback tests in flowcb/accept * Bunch more callback tests in flowcb/accept * Add more tests for flowcb/accept * Fix tests for flowcb/accept/tohttp THey were broken because of the 'set_notice' header being wrong * More flowcb/accept tests!! flowcb/accept/postoverride flowcb/accept/tohttp flowcb/accept/wmotypesuffix * Remove set_notice items in message That item was removed as part of writing unit tests * Bunch more callback tests in flowcb/accept * Add more tests for flowcb/accept * Fix tests for flowcb/accept/tohttp THey were broken because of the 'set_notice' header being wrong * More flowcb/accept tests!! flowcb/accept/postoverride flowcb/accept/tohttp flowcb/accept/wmotypesuffix * Remove set_notice items in message That item was removed as part of writing unit tests * Bunch more callback tests in flowcb/accept * Add more tests for flowcb/accept * Fix tests for flowcb/accept/tohttp THey were broken because of the 'set_notice' header being wrong * More flowcb/accept tests!! flowcb/accept/postoverride flowcb/accept/tohttp flowcb/accept/wmotypesuffix * Remove set_notice items in message That item was removed as part of writing unit tests * Bunch more callback tests in flowcb/accept * Remove set_notice items in message That item was removed as part of writing unit tests * More flowcb/accept tests downloadbaseurl renamedmf tolocalfile All with 100% coverage * Yes more flowcb/accept tests * Final set of flowcb/accept unit tests * Bunch more callback tests in flowcb/accept * Add more tests for flowcb/accept * Fix tests for flowcb/accept/tohttp THey were broken because of the 'set_notice' header being wrong * More flowcb/accept tests!! flowcb/accept/postoverride flowcb/accept/tohttp flowcb/accept/wmotypesuffix * Remove set_notice items in message That item was removed as part of writing unit tests * Bunch more callback tests in flowcb/accept * Remove set_notice items in message That item was removed as part of writing unit tests * More flowcb/accept tests downloadbaseurl renamedmf tolocalfile All with 100% coverage * Yes more flowcb/accept tests * Final set of flowcb/accept unit tests * Final fixes for flowcb/accept plugins Should have been part of the inital set of fixes, but they weren't known until the tests were fixed.. sort of circular problem. * Update test config There's a class we're testing that starts with 'Test', and it was throwing warnings for pytest. Updating the config to only look for classes that start with 'Test_*' now. * Fixes to all flowcb/accept tests These fixes reflect the changes that were done in #756 Some of them also are fixes to make sure these tests are successful when run as part of the whole suite (caplog message counts among them). Those assertions probably aren't all that purposeful, and checking that the desired logs are in the list is better, but here we are..
MetPX · Sep 7, 2023 · 58fbbba · 58fbbba
1 parent 47598d3
commit 58fbbba
Show file tree

Hide file tree

Showing 27 changed files with 1,486 additions and 8 deletions.
diff --git a/sarracenia/flowcb/accept/tohttp.py b/sarracenia/flowcb/accept/tohttp.py
@@ -37,13 +37,13 @@ def after_accept(self, worklist):
 
             url = urllib.parse.urlparse(message['baseUrl'])
 
-            new_baseUrl = 'http://'
+            new_baseUrl = 'http://' + url.netloc
             if self._ldocroot != None:
                 new_baseUrl += self._ldocroot
 
-            new_baseUrl += url.path.replace('///', '//')
+            new_baseUrl += url.path
 
-            message['baseUrl'] = new_baseUrl
+            message['baseUrl'] = new_baseUrl.replace('///', '//')
 
-            logger.debug("ToHttp config; baseDir=%s, toHttpRoot" % (self.o.baseDir, self.o.toHttpRoot))
+            logger.debug("ToHttp config; baseDir=%s, toHttpRoot=%s" % (self.o.baseDir, self.o.toHttpRoot))
             logger.info("ToHttp message output: baseUrl=%s, relPath=%s" % (message['baseUrl'], message['relPath']))
diff --git a/sarracenia/flowcb/accept/tolocal.py b/sarracenia/flowcb/accept/tolocal.py
@@ -54,7 +54,9 @@ class ToLocal(FlowCB):
     def __init__(self, options):
         super().__init__(options,logger)
 
-        if hasattr(self.o, 'baseDir'):
+        self._ldocroot = None
+
+        if self.o.baseDir:
             self._ldocroot = self.o.baseDir
 
         self.o.add_option('toLocalRoot', 'str')

diff --git a/sarracenia/flowcb/accept/wmotypesuffix.py b/sarracenia/flowcb/accept/wmotypesuffix.py
@@ -44,9 +44,9 @@ def __find_type(self, TT):
     def after_accept(self, worklist):
         for message in worklist.incoming:
             type_suffix = self.__find_type(message['new_file'][0:2])
-            # FIXME confused as to how this could ever be true since find_type never returns "UNKNOWN"
-            if type_suffix == 'UNKNOWN':
-                continue
+            ## FIXME confused as to how this could ever be true since find_type never returns "UNKNOWN"
+            #if type_suffix == 'UNKNOWN':
+            #    continue
 
             # file name already has suffix
             if message['new_file'][-len(type_suffix):] == type_suffix:

diff --git a/tests/pytest.ini b/tests/pytest.ini
@@ -7,5 +7,6 @@ minversion = 7.0
 norecursedirs = obsolete docs debian docker tools
 python_files = *_test.py
 python_functions = test_*
+python_classes = Test_*
 log_cli = False
 testpaths = tests
diff --git a/tests/sarracenia/flowcb/accept/dateappend_test.py b/tests/sarracenia/flowcb/accept/dateappend_test.py
@@ -0,0 +1,43 @@
+import pytest
+import types, re
+
+
+#useful for debugging tests
+def pretty(*things, **named_things):
+    import pprint
+    for t in things:
+        pprint.PrettyPrinter(indent=2, width=200).pprint(t)
+    for k,v in named_things.items():
+        print(str(k) + ":")
+        pprint.PrettyPrinter(indent=2, width=200).pprint(v)
+
+from sarracenia.flowcb.accept.dateappend import Dateappend
+from sarracenia import Message as SR3Message
+import sarracenia.config
+
+def make_message():
+    m = SR3Message()
+    m["new_file"] = './SK/s0000684_f.xml'
+
+    return m
+
+def make_worklist():
+    WorkList = types.SimpleNamespace()
+    WorkList.ok = []
+    WorkList.incoming = []
+    WorkList.rejected = []
+    WorkList.failed = []
+    WorkList.directories_ok = []
+    return WorkList
+
+def test_after_accept():
+    dateappend = Dateappend(sarracenia.config.default_config())
+
+    worklist = make_worklist()
+    worklist.incoming = [make_message(), make_message()]
+
+    dateappend.after_accept(worklist)
+
+    assert len(worklist.incoming) == 2
+    assert bool(re.match(r'./SK/s0000684_f.xml_\d{12}', worklist.incoming[0]['new_file'])) == True
+    assert bool(re.match(r'./SK/s0000684_f.xml_\d{12}', worklist.incoming[1]['new_file'])) == True
diff --git a/tests/sarracenia/flowcb/accept/delete_test.py b/tests/sarracenia/flowcb/accept/delete_test.py
@@ -0,0 +1,79 @@
+import pytest
+import types
+import os
+
+#useful for debugging tests
+def pretty(*things, **named_things):
+    import pprint
+    for t in things:
+        pprint.PrettyPrinter(indent=2, width=200).pprint(t)
+    for k,v in named_things.items():
+        print(str(k) + ":")
+        pprint.PrettyPrinter(indent=2, width=200).pprint(v)
+
+from sarracenia.flowcb.accept.delete import Delete
+from sarracenia import Message as SR3Message
+import sarracenia.config
+
+class dummy_consumer:
+    def __init__(self):
+        self.sleep_now = 10
+        self.sleep_min = 0
+
+    def msg_to_retry(self):
+        pass
+
+def make_message(dir, file):
+    m = SR3Message()
+    m['new_dir'] = dir
+    m['new_file'] = file
+    return m
+
+def make_worklist():
+    WorkList = types.SimpleNamespace()
+    WorkList.ok = []
+    WorkList.incoming = []
+    WorkList.rejected = []
+    WorkList.failed = []
+    WorkList.directories_ok = []
+    return WorkList
+
+def test___init__():
+    options = sarracenia.config.default_config()
+    options.logLevel = 'DEBUG'
+    deletecb = Delete(options)
+
+
+@pytest.mark.depends(on=['test___init__'])
+def test_after_accept(tmp_path, caplog):
+    file = str(tmp_path) + os.sep + 'cfr/file.txt'
+    file_c = str(tmp_path) + os.sep + 'cfile/file.txt'
+    #os.mkdir(str(tmp_path) + os.sep + 'cfr')
+    #os.mkdir(str(tmp_path) + os.sep + 'cfile')
+
+    options = sarracenia.config.default_config()
+    options.logLevel = "DEBUG"
+    options.consumer = dummy_consumer()
+    deletecb = Delete(options)
+
+    caplog.clear()
+    worklist = make_worklist()
+    worklist.incoming = [make_message(str(tmp_path), 'cfr/file.txt')]
+    # Should be able to catch that it's raising an error, but no matter what error I tell it's suppose to raise, it doesn't match
+    #with pytest.raises(FileNotFoundError):
+    deletecb.after_accept(worklist)
+    assert len(caplog.messages) == 2
+    assert len(worklist.incoming) == 0
+    assert len(worklist.rejected) == 1
+    assert f'could not unlink {file}: [Errno 2] No such file or directory: \'{file}\'' in caplog.messages
+
+    caplog.clear()
+    worklist = make_worklist()
+    worklist.incoming = [make_message(str(tmp_path), 'cfr/file.txt')]
+    os.mkdir(str(tmp_path) + os.sep + 'cfr')
+    os.mkdir(str(tmp_path) + os.sep + 'cfile')
+    open(file, 'a').close()
+    open(file_c, 'a').close()
+    deletecb.after_accept(worklist)
+    assert len(worklist.incoming) == 1
+    assert f'deleted: {file} and the cfile version.' in caplog.messages
diff --git a/tests/sarracenia/flowcb/accept/downloadbaseurl_test.py b/tests/sarracenia/flowcb/accept/downloadbaseurl_test.py
@@ -0,0 +1,54 @@
+import pytest
+import types
+
+import os
+
+#useful for debugging tests
+def pretty(*things, **named_things):
+    import pprint
+    for t in things:
+        pprint.PrettyPrinter(indent=2, width=200).pprint(t)
+    for k,v in named_things.items():
+        print(str(k) + ":")
+        pprint.PrettyPrinter(indent=2, width=200).pprint(v)
+
+from sarracenia.flowcb.accept.downloadbaseurl import DownloadBaseUrl
+from sarracenia import Message as SR3Message
+import sarracenia.config
+
+def make_message(fileNumber = 1):
+    m = SR3Message()
+    m['baseUrl'] = 'http://NotAReal.url/a/rel/Path/file.txt'
+    m['new_file'] = f"/file{fileNumber}.txt"
+
+    return m
+
+def make_worklist():
+    WorkList = types.SimpleNamespace()
+    WorkList.ok = []
+    WorkList.incoming = []
+    WorkList.rejected = []
+    WorkList.failed = []
+    WorkList.directories_ok = []
+    return WorkList
+
+def test___init__():
+    options = sarracenia.config.default_config()
+    downloadbaseurl = DownloadBaseUrl(options)
+
+
+def test_after_accept(tmp_path, mocker):
+    import io
+    mocker.patch('urllib.request.urlopen', return_value=io.BytesIO(b'SomeRandomData'))
+
+    new_dir = str(tmp_path) + os.sep + 'new_dir'
+    options = sarracenia.config.default_config()
+    options.new_dir = new_dir
+    downloadbaseurl = DownloadBaseUrl(options)
+
+    #Set 1 - option.new_dir doesn't exists
+    worklist = make_worklist()
+    worklist.incoming = [make_message(1)]
+    downloadbaseurl.after_accept(worklist)
+    assert len(worklist.incoming) == 1
+    assert open(new_dir + '/file1.txt', 'r').read() == "SomeRandomData"
diff --git a/tests/sarracenia/flowcb/accept/hourtree_test.py b/tests/sarracenia/flowcb/accept/hourtree_test.py
@@ -0,0 +1,42 @@
+import pytest
+import types, re
+
+#useful for debugging tests
+def pretty(*things, **named_things):
+    import pprint
+    for t in things:
+        pprint.PrettyPrinter(indent=2, width=200).pprint(t)
+    for k,v in named_things.items():
+        print(str(k) + ":")
+        pprint.PrettyPrinter(indent=2, width=200).pprint(v)
+
+from sarracenia.flowcb.accept.hourtree import HourTree
+from sarracenia import Message as SR3Message
+import sarracenia.config
+
+def make_message():
+    m = SR3Message()
+    m['new_file'] = '/foo/bar/NewFile.txt'
+    m['new_dir'] = '/foo/bar'
+
+    return m
+
+def make_worklist():
+    WorkList = types.SimpleNamespace()
+    WorkList.ok = []
+    WorkList.incoming = []
+    WorkList.rejected = []
+    WorkList.failed = []
+    WorkList.directories_ok = []
+    return WorkList
+
+def test_after_accept():
+    hourtree = HourTree(sarracenia.config.default_config())
+
+    worklist = make_worklist()
+    worklist.incoming = [make_message(), make_message()]
+
+    hourtree.after_accept(worklist)
+    assert len(worklist.incoming) == 2
+    assert bool(re.match(r"/foo/bar/\d{2}", worklist.incoming[0]['new_dir'])) == True
+    assert bool(re.match(r"/foo/bar/\d{2}/NewFile.txt", worklist.incoming[1]['new_file'])) == True
diff --git a/tests/sarracenia/flowcb/accept/httptohttps_test.py b/tests/sarracenia/flowcb/accept/httptohttps_test.py
@@ -0,0 +1,50 @@
+import pytest
+import types, re
+
+
+#useful for debugging tests
+def pretty(*things, **named_things):
+    import pprint
+    for t in things:
+        pprint.PrettyPrinter(indent=2, width=200).pprint(t)
+    for k,v in named_things.items():
+        print(str(k) + ":")
+        pprint.PrettyPrinter(indent=2, width=200).pprint(v)
+
+from sarracenia.flowcb.accept.httptohttps import HttpToHttps
+from sarracenia import Message as SR3Message
+import sarracenia.config
+
+def make_message(scheme):
+    m = SR3Message()
+    m['baseUrl'] = scheme + '://NotAReal.url'
+    m['relPath'] = 'a/rel/Path/file.txt'
+    m['pubTime'] = '20180118151049.356378078'
+
+    return m
+
+def make_worklist():
+    WorkList = types.SimpleNamespace()
+    WorkList.ok = []
+    WorkList.incoming = []
+    WorkList.rejected = []
+    WorkList.failed = []
+    WorkList.directories_ok = []
+    return WorkList
+
+def test_after_accept():
+    httptohttps = HttpToHttps(sarracenia.config.default_config())
+
+    message_http = make_message('http')
+    message_https = make_message('https')
+    message_sftp = make_message('sftp')
+
+    worklist = make_worklist()
+    worklist.incoming = [message_http, message_https, message_sftp]
+
+    httptohttps.after_accept(worklist)
+    assert len(worklist.incoming) == 3
+    assert worklist.incoming[0]['baseUrl'] == 'https://NotAReal.url'
+    assert worklist.incoming[1]['baseUrl'] == 'https://NotAReal.url'
+    assert worklist.incoming[2]['baseUrl'] == 'sftp://NotAReal.url'
+    #assert 'set_notice' not in worklist.incoming[2]
diff --git a/tests/sarracenia/flowcb/accept/longflow_test.py b/tests/sarracenia/flowcb/accept/longflow_test.py
@@ -0,0 +1,41 @@
+import pytest
+import types, re
+
+
+#useful for debugging tests
+def pretty(*things, **named_things):
+    import pprint
+    for t in things:
+        pprint.PrettyPrinter(indent=2, width=200).pprint(t)
+    for k,v in named_things.items():
+        print(str(k) + ":")
+        pprint.PrettyPrinter(indent=2, width=200).pprint(v)
+
+from sarracenia.flowcb.accept.longflow import LongFlow
+from sarracenia import Message as SR3Message
+import sarracenia.config
+
+def make_message():
+    m = SR3Message()
+    m['headers'] = {}
+
+    return m
+
+def make_worklist():
+    WorkList = types.SimpleNamespace()
+    WorkList.ok = []
+    WorkList.incoming = []
+    WorkList.rejected = []
+    WorkList.failed = []
+    WorkList.directories_ok = []
+    return WorkList
+
+def test_after_accept():
+    longflow = LongFlow(sarracenia.config.default_config())
+
+    worklist = make_worklist()
+    worklist.incoming = [make_message(), make_message()]
+
+    longflow.after_accept(worklist)
+    assert len(worklist.incoming) == 2
+    assert len(worklist.incoming[0]['toolong']) == len('1234567890ßñç' * 26)