From 5ae40a71a7380c8be12dcee2ff1f4eb4ac94f01c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C5=A0=C4=8Dotka?= Date: Mon, 24 May 2021 13:57:22 +0200 Subject: [PATCH] add there important improvement of file nadling --- fmf/base.py | 29 +++++++------------ fmf/plugin_loader.py | 35 ++++++++++++++++++---- fmf/plugins/bash.py | 2 +- fmf/plugins/pytest/plugin.py | 4 +-- fmf/utils.py | 37 ++++++++++++++++++++++++ tests/tests_plugin/test_basic.py | 1 + tests/tests_plugin/test_rewrite.fmf | 4 +++ tests/tests_plugin/test_rewrite.py | 7 +++++ tests/unit/test_plugin.py | 45 ++++++++++++++++++++++++++++- 9 files changed, 137 insertions(+), 27 deletions(-) create mode 100644 tests/tests_plugin/test_rewrite.fmf create mode 100644 tests/tests_plugin/test_rewrite.py diff --git a/fmf/base.py b/fmf/base.py index f02adeb2..95bcb08e 100644 --- a/fmf/base.py +++ b/fmf/base.py @@ -19,7 +19,7 @@ import fmf.utils as utils from io import open -from fmf.utils import log, dict_to_yaml +from fmf.utils import log, dict_to_yaml, FileSorting from fmf.constants import SUFFIX, IGNORED_DIRECTORIES, MAIN from fmf.plugin_loader import get_suffixes, get_plugin_for_file from pprint import pformat as pretty @@ -452,15 +452,11 @@ def grow(self, path): except StopIteration: log.debug("Skipping '{0}' (not accessible).".format(path)) return - # Investigate main.fmf as the first file (for correct inheritance) - filenames = sorted( - [filename for filename in filenames if any(filter(filename.endswith, get_suffixes()))]) - try: - filenames.insert(0, filenames.pop(filenames.index(MAIN))) - except ValueError: - pass + + filenames_sorted = sorted( + [FileSorting(filename) for filename in filenames if any(filter(filename.endswith, get_suffixes()))]) # Check every metadata file and load data (ignore hidden) - for filename in filenames: + for filename in [filename.value for filename in filenames_sorted]: if filename.startswith("."): continue fullpath = os.path.abspath(os.path.join(dirpath, filename)) @@ -474,16 +470,13 @@ def grow(self, path): raise (utils.FileError("Failed to parse '{0}'.\n{1}".format( fullpath, error))) else: + data = None plugin = get_plugin_for_file(fullpath) log.debug("Used plugin {}".format(plugin)) - if plugin.file_patters and any(filter(lambda x: re.search(x, fullpath), plugin.file_patters)): - log.info("Matched patters {}".format(plugin.file_patters)) - data = plugin().get_data(fullpath) - # ignore results of output if there is None - if data is None: - continue - else: - log.debug("Does not match patters {}".format(plugin.file_patters)) + if plugin: + data = plugin().read(fullpath) + # ignore results of output if there is None + if data is None: continue log.data(pretty(data)) # Handle main.fmf as data for self @@ -734,7 +727,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): with open(source, "w", encoding='utf-8') as file: file.write(dict_to_yaml(full_data)) else: - plugin().put_data(source, hierarchy, node_data, append, modified, deleted) + plugin().write(source, hierarchy, node_data, append, modified, deleted) def __getitem__(self, key): """ diff --git a/fmf/plugin_loader.py b/fmf/plugin_loader.py index a9186577..57a03107 100644 --- a/fmf/plugin_loader.py +++ b/fmf/plugin_loader.py @@ -1,9 +1,13 @@ from functools import lru_cache import inspect +import copy +import yaml + from fmf.constants import PLUGIN_ENV, SUFFIX from fmf.utils import log import importlib import os +import re class Plugin: @@ -14,19 +18,40 @@ class Plugin: extensions = list() file_patters = list() - def get_data(self, filename): + def read(self, filename): """ return python dictionary representation of metadata inside file (FMF structure) """ raise NotImplementedError("Define own impementation") - def put_data( + @staticmethod + def __define_undefined(hierarchy, modified, append): + output = dict() + current = output + for key in hierarchy: + if key not in current or current[key] is None: + current[key] = dict() + current = current[key] + for k,v in modified.items(): + current[k] = v + for k,v in append.items(): + current[k] = v + return output + + def write( self, filename, hierarchy, data, append_dict, modified_dict, deleted_items): """ - Write data in dictionary representation back to file + Write data in dictionary representation back to file, if not defined, create new fmf file with same name. + When created, nodes will not use plugin method anyway """ - raise NotImplementedError("Define own impementation") + path = os.path.dirname(filename) + basename = os.path.basename(filename) + current_extension = list(filter(lambda x: basename.endswith(x), self.extensions))[0] + without_extension = basename[0:-len(list(current_extension))] + fmf_file = os.path.join(path, without_extension + ".fmf") + with open(fmf_file, "w") as fd: + yaml.safe_dump(self.__define_undefined(hierarchy, modified_dict, append_dict), stream=fd) @lru_cache(maxsize=1) @@ -61,6 +86,6 @@ def get_suffixes(): def get_plugin_for_file(filename): extension = "." + filename.rsplit(".", 1)[1] for item in enabled_plugins(): - if extension in item.extensions: + if extension in item.extensions and any(filter(lambda x: re.search(x, filename), item.file_patters)): log.debug("File {} parsed by by plugin {}".format(filename, item)) return item diff --git a/fmf/plugins/bash.py b/fmf/plugins/bash.py index 0fbefc40..5ea96cbc 100644 --- a/fmf/plugins/bash.py +++ b/fmf/plugins/bash.py @@ -21,6 +21,6 @@ def update_data(filename, pattern="^#.*:FMF:"): out[identifier] = value return out - def get_data(self, file_name): + def read(self, file_name): log.info("Processing Item: {}".format(file_name)) return self.update_data(file_name) diff --git a/fmf/plugins/pytest/plugin.py b/fmf/plugins/pytest/plugin.py index 891acad4..45dbbf08 100644 --- a/fmf/plugins/pytest/plugin.py +++ b/fmf/plugins/pytest/plugin.py @@ -234,7 +234,7 @@ def update_data(store_dict, func, config): define_undefined(store_dict, keys, config, filename, cls, test) return store_dict - def get_data(self, file_name): + def read(self, file_name): def call_collect(queue, file_name): """ have to call in separate process, to avoid problems with pytest multiple collectitons @@ -268,7 +268,7 @@ def import_test_module(filename): loader.exec_module(module) return module - def put_data( + def write( self, filename, hierarchy, data, append_dict, modified_dict, deleted_items): module = self.import_test_module(filename) diff --git a/fmf/utils.py b/fmf/utils.py index cca040ec..9dbe1d98 100644 --- a/fmf/utils.py +++ b/fmf/utils.py @@ -25,6 +25,7 @@ from io import StringIO import fmf.base +from fmf.constants import MAIN # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Constants @@ -856,3 +857,39 @@ def representer(self, data): return self.represent_mapping( return output.getvalue().decode('utf-8') except AttributeError: return output.getvalue() + + +class FileSorting: + def __init__(self, value): + self._value = value + + @property + def value(self): + return self._value + + @property + def splitted(self): + splitted = self._value.rsplit(".", 1) + basename = splitted[0] + suffix = splitted[1] if len(splitted) > 1 else '' + return basename, suffix + + @property + def basename(self): + return self.splitted[0] + + @property + def suffix(self): + return self.splitted[1] + + def __lt__(self, other): + # Investigate main.fmf as the first file (for correct inheritance) + if self.value == MAIN: + return True + # if there are same filenames and other endswith fmf, it has to be last one + elif self.basename == other.basename and other.suffix == "fmf": + return True + elif self.basename == other.basename and self.suffix == "fmf": + return False + else: + return self.value < other.value diff --git a/tests/tests_plugin/test_basic.py b/tests/tests_plugin/test_basic.py index 57eb9a1c..385ea5c0 100644 --- a/tests/tests_plugin/test_basic.py +++ b/tests/tests_plugin/test_basic.py @@ -14,6 +14,7 @@ def test_fail(): assert False +@TMT.summary("Some summary") @pytest.mark.skip def test_skip(): assert True diff --git a/tests/tests_plugin/test_rewrite.fmf b/tests/tests_plugin/test_rewrite.fmf new file mode 100644 index 00000000..5efa8729 --- /dev/null +++ b/tests/tests_plugin/test_rewrite.fmf @@ -0,0 +1,4 @@ +/test_pass: + added_fmf_file: added + summary: Rewrite + tag+: [tier2] \ No newline at end of file diff --git a/tests/tests_plugin/test_rewrite.py b/tests/tests_plugin/test_rewrite.py new file mode 100644 index 00000000..b3413529 --- /dev/null +++ b/tests/tests_plugin/test_rewrite.py @@ -0,0 +1,7 @@ +from fmf.plugins.pytest import TMT + + +@TMT.tag("Tier1") +@TMT.summary("Rewritten") +def test_pass(): + assert True diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index 7e5db2c9..99041d7c 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -35,8 +35,13 @@ def setUp(self): os.environ[PLUGIN_ENV] = "fmf.plugins.pytest" self.plugin_tree = Tree(self.tempdir) + def tearDown(self): + enabled_plugins.cache_clear() + #rmtree(self.tempdir) + def test_basic(self): item = self.plugin_tree.find("/test_basic/test_skip") + self.assertFalse(item.data.get("enabled")) self.assertIn("Jan", item.data["author"]) self.assertIn( @@ -63,6 +68,28 @@ def test_modify(self): self.assertIn("tier2", item.data["tag"]) self.assertNotIn("tier", item.data) + def test_rewrite(self): + item = self.plugin_tree.find("/test_rewrite/test_pass") + self.assertNotIn("duration", item.data) + self.assertIn("Tier1", item.data["tag"]) + self.assertIn("tier2", item.data["tag"]) + self.assertEqual("added", item.data["added_fmf_file"]) + self.assertEqual("Rewrite", item.data["summary"]) + + def test_rewrite_modify(self): + self.test_rewrite() + item = self.plugin_tree.find("/test_rewrite/test_pass") + with item as data: + data["tag+"] += ["tier3"] + data["extra_id"] = 1234 + + self.plugin_tree = Tree(self.tempdir) + item = self.plugin_tree.find("/test_rewrite/test_pass") + self.test_rewrite() + self.assertEqual(1234, item.data["extra_id"]) + self.assertIn("tier3", item.data["tag"]) + + class Bash(Base): """ Verify reading data done via plugins """ @@ -73,8 +100,24 @@ def setUp(self): os.path.join(PLUGIN_PATH, "bash.py")) self.plugin_tree = Tree(self.tempdir) - def test_pytest_plugin(self): + def test_read(self): item = self.plugin_tree.find("/runtest") self.assertIn("tier1", item.data["tag"]) self.assertIn("./runtest.sh", item.data["test"]) self.assertIn("Jan", item.data["author"]) + + def test_modify(self): + self.assertNotIn("runtest.fmf", os.listdir(self.tempdir)) + item = self.plugin_tree.find("/runtest") + self.assertIn("tier1", item.data["tag"]) + self.assertIn("./runtest.sh", item.data["test"]) + with item as data: + data["tier"] = 0 + data["duration"] = "10m" + self.plugin_tree = Tree(self.tempdir) + item = self.plugin_tree.find("/runtest") + self.assertIn("runtest.fmf", os.listdir(self.tempdir)) + self.assertEqual("10m", item.data["duration"]) + self.assertEqual(0, item.data["tier"]) + self.assertIn("tier1", item.data["tag"]) + self.assertIn("./runtest.sh", item.data["test"])