Add files via upload

Tools/ss14_ru/file.py

@@ -0,0 +1,91 @@
+import typing
+
+from fluent.syntax import ast
+from yamlmodels import YAMLElements
+import os
+
+
+class File:
+    def __init__(self, full_path):
+        self.full_path = full_path
+
+    def read_data(self):
+        file = open(self.full_path, 'r', encoding='utf8')
+        # replace необходим для того, чтобы 1-е сообщение не считалось ast.Junk
+        file_data = file.read().replace('', '')
+        file.close()
+
+        return file_data
+
+    def save_data(self, file_data: typing.AnyStr):
+        os.makedirs(os.path.dirname(self.full_path), exist_ok=True)
+        file = open(self.full_path, 'w', encoding='utf8')
+        file.write(file_data)
+        file.close()
+
+    def get_relative_path(self, base_path):
+        return os.path.relpath(self.full_path, base_path)
+
+    def get_relative_path_without_extension(self, base_path):
+        return self.get_relative_path(base_path).split('.', maxsplit=1)[0]
+
+    def get_relative_parent_dir(self, base_path):
+        return os.path.relpath(self.get_parent_dir(), base_path)
+
+    def get_parent_dir(self):
+        return os.path.dirname(self.full_path)
+
+    def get_name(self):
+        return os.path.basename(self.full_path).split('.')[0]
+
+
+class FluentFile(File):
+    def __init__(self, full_path):
+        super().__init__(full_path)
+        self.full_path = full_path
+
+    def parse_data(self, file_data: typing.AnyStr):
+        from fluent.syntax import FluentParser
+
+        return FluentParser().parse(file_data)
+
+    def serialize_data(self, parsed_file_data: ast.Resource):
+        from fluent.syntax import FluentSerializer
+
+        return FluentSerializer(with_junk=True).serialize(parsed_file_data)
+
+    def read_serialized_data(self):
+        return self.serialize_data(self.parse_data(self.read_data()))
+
+    def read_parsed_data(self):
+        return self.parse_data(self.read_data())
+
+
+class YAMLFluentFileAdapter(File):
+    def __init__(self, full_path):
+        super().__init__(full_path)
+        self.full_path = full_path
+
+    # def create_fluent_from_yaml_elements(self, yaml_elements):
+
+
+
+class YAMLFile(File):
+    def __init__(self, full_path):
+        super().__init__(full_path)
+
+    def parse_data(self, file_data: typing.AnyStr):
+        import yaml
+
+        return yaml.load(file_data, Loader=yaml.BaseLoader)
+
+    def get_elements(self, parsed_data):
+
+        if isinstance(parsed_data, list):
+            elements = YAMLElements(parsed_data).elements
+            # элемент может быть None, если имеет неизвестный тип
+            exist_elements = list(filter(lambda el: el, elements))
+
+            return exist_elements
+
+        return []

Tools/ss14_ru/fluentast.py

@@ -0,0 +1,187 @@
+import typing
+
+from fluent.syntax import ast, FluentParser, FluentSerializer
+from lokalisemodels import LokaliseKey
+from pydash import py_
+
+
+class FluentAstAbstract:
+    element = None
+    @classmethod
+    def get_id_name(cls, element):
+        if isinstance(element, ast.Junk):
+            return FluentAstJunk(element).get_id_name()
+        elif isinstance(element, ast.Message):
+            return FluentAstMessage(element).get_id_name()
+        elif isinstance(element, ast.Term):
+            return FluentAstTerm(element).get_id_name()
+        else:
+            return None
+
+    @classmethod
+    def create_element(cls, element):
+        if isinstance(element, ast.Junk):
+            cls.element = FluentAstJunk(element)
+            return cls.element
+        elif isinstance(element, ast.Message):
+            cls.element = FluentAstMessage(element)
+            return cls.element
+        elif isinstance(element, ast.Term):
+            cls.element = FluentAstTerm(element)
+            return cls.element
+        else:
+            return None
+
+
+class FluentAstMessage:
+    def __init__(self, message: ast.Message):
+        self.message = message
+        self.element = message
+
+    def get_id_name(self):
+        return self.message.id.name
+
+
+class FluentAstTerm:
+    def __init__(self, term: ast.Term):
+        self.term = term
+        self.element = term
+
+    def get_id_name(self):
+        return self.term.id.name
+
+
+class FluentAstAttribute:
+    def __init__(self, id, value, parent_key = None):
+        self.id = id
+        self.value = value
+        self.parent_key = parent_key
+
+
+class FluentAstAttributeFactory:
+    @classmethod
+    def from_yaml_element(cls, element):
+        attrs = []
+        if element.description:
+            attrs.append(FluentAstAttribute('desc', element.description))
+
+        if element.suffix:
+            attrs.append(FluentAstAttribute('suffix', element.suffix))
+
+        if not len(attrs):
+            return None
+
+        return attrs
+
+
+class FluentAstJunk:
+    def __init__(self, junk: ast.Junk):
+        self.junk = junk
+        self.element = junk
+
+    def get_id_name(self):
+        return self.junk.content.split('=')[0].strip()
+
+
+class FluentSerializedMessage:
+    @classmethod
+    def from_yaml_element(cls, id, value, attributes, parent_id = None, raw_key = False):
+        if not value and not id and not parent_id:
+            return None
+
+        if not attributes:
+            attributes = []
+
+        if len(list(filter(lambda attr: attr.id == 'desc', attributes))) == 0:
+            if parent_id:
+                attributes.append(FluentAstAttribute('desc', '{ ' + FluentSerializedMessage.get_key(parent_id) + '.desc' + ' }'));
+            else:
+                attributes.append(FluentAstAttribute('desc', '{ "" }'))
+
+        message = f'{cls.get_key(id, raw_key)} = {cls.get_value(value, parent_id)}\n'
+
+        if attributes and len(attributes):
+            full_message = message
+
+            for attr in attributes:
+                fluent_newlines = attr.value.replace("\n", "\n        ");
+                full_message = cls.add_attr(full_message, attr.id, fluent_newlines, raw_key=raw_key)
+
+            desc_attr = py_.find(attributes, lambda a: a.id == 'desc')
+            if not desc_attr and parent_id:
+                full_message = cls.add_attr(full_message, 'desc', '{ ' + FluentSerializedMessage.get_key(parent_id) + '.desc' + ' }')
+
+            return full_message
+
+        return cls.to_serialized_message(message)
+
+    @classmethod
+    def from_lokalise_keys(cls, keys: typing.List[LokaliseKey]):
+        attributes_keys = list(filter(lambda k: k.is_attr, keys))
+        attributes = list(map(lambda k: FluentAstAttribute(id='.{name}'.format(name=k.get_key_last_name(k.key_name)),
+                                                           value=FluentSerializedMessage.get_attr(k, k.get_key_last_name(k.key_name)), parent_key=k.get_parent_key()),
+                              attributes_keys))
+        attributes_group = py_.group_by(attributes, 'parent_key')
+
+        serialized_message = ''
+        for key in keys:
+            if key.is_attr:
+                continue
+            key_name = key.get_key_last_name(key.key_name)
+            key_value = key.get_translation('ru').data['translation']
+            key_attributes = []
+
+            if len(attributes_group):
+                k = f'{key.get_key_base_name(key.key_name)}.{key_name}'
+                key_attributes = attributes_group[k] if k in attributes_group else []
+
+            message = key.serialize_message()
+            full_message = cls.from_yaml_element(key_name, key_value, key_attributes, key.get_parent_key(), True)
+
+            if full_message:
+                serialized_message = serialized_message + '\n' + full_message
+            elif message:
+                serialized_message = serialized_message + '\n' + message
+            else:
+                raise Exception('Что-то пошло не так')
+
+        return serialized_message
+
+    @staticmethod
+    def get_attr(k, name, parent_id = None):
+        if parent_id:
+            return "{ " + parent_id + f'.{name}' + " }"
+        else:
+            return k.get_translation('ru').data['translation']
+
+
+    @staticmethod
+    def to_serialized_message(string_message):
+        if not string_message:
+            return None
+
+        ast_message = FluentParser().parse(string_message)
+        serialized = FluentSerializer(with_junk=True).serialize(ast_message)
+
+        return serialized if serialized else ''
+
+    @staticmethod
+    def add_attr(message_str, attr_key, attr_value, raw_key = False):
+        prefix = '' if raw_key else '.'
+        return f'{message_str}\n  {prefix}{attr_key} = {attr_value}'
+
+    @staticmethod
+    def get_value(value, parent_id):
+        if value:
+            return value
+        elif parent_id:
+            return '{ ' + FluentSerializedMessage.get_key(parent_id) + ' }'
+        else:
+            return '{ "" }'
+
+    @staticmethod
+    def get_key(id, raw = False):
+        if raw:
+            return f'{id}'
+        else:
+            return f'ent-{id}'

Tools/ss14_ru/fluentastcomparer.py

@@ -0,0 +1,100 @@
+from fluent.syntax import ast
+from fluentast import FluentAstAbstract
+from pydash import py_
+
+
+class FluentAstComparer:
+    def __init__(self, sourse_parsed: ast.Resource, target_parsed: ast.Resource):
+        self.sourse_parsed = sourse_parsed
+        self.target_parsed = target_parsed
+        self.source_elements = list(
+            filter(lambda el: el, list(map(lambda e: FluentAstAbstract.create_element(e), sourse_parsed.body))))
+        self.target_elements = list(
+            filter(lambda el: el, list(map(lambda e: FluentAstAbstract.create_element(e), target_parsed.body))))
+
+    # Возвращает полностью эквивалентные сообщения (не считая span)
+    def get_equal_elements(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span'])
+
+        return py_.intersection_with(self.source_elements, self.target_elements, comparator=comparator)
+
+    # Возвращает полностью неэквивалентные сообщения (не считая span)
+    def get_not_equal_elements(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span'])
+        diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения с эквивалентными именами ключей
+    def get_equal_id_names(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        eq = py_.intersection_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return eq
+
+    # Возвращает сообщения с неэквивалентными именами ключей
+    def get_not_equal_id_names(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения target, существующие в source
+    def get_exist_id_names(self, source, target):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        eq = py_.intersection_with(source, target, comparator=comparator)
+
+        return eq
+
+    # Возвращает сообщения target, существующие в source
+    def get_not_exist_id_names(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes'])
+        diff = py_.difference_with(self.target_elements, self.source_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения с эквивалентным значением и атрибутами
+    def get_equal_values_with_attrs(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment'])
+        eq = py_.intersection_with(self.target_elements, self.source_elements, comparator=comparator)
+
+        return eq
+
+    # Возвращает сообщения из source с неэквивалентным значением и атрибутами
+    def get_not_equal_values_with_attrs(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment'])
+        diff = py_.difference_with(self.source_elements, self.target_elements,
+                                   comparator=lambda a, b: a.element.equals(b.element,
+                                                                            ignored_fields=['span', 'id', 'comment']))
+
+        return diff
+
+    # Возвращает сообщения из source, существующие в target и source, с неэквивалентным значением и атрибутами
+    def get_not_equal_exist_values_with_attrs(self):
+        diff = py_.difference_with(self.source_elements, self.target_elements,
+                                   comparator=lambda a, b: a.element.equals(b.element,
+                                                                            ignored_fields=['span', 'id', 'comment']))
+        ex = self.get_exist_id_names(self.source_elements, self.target_elements)
+        exist = py_.intersection(diff, ex)
+
+        return exist
+
+        # Возвращает сообщения из target с неэквивалентным значением и атрибутами
+
+    def get_target_not_equal_values_with_attrs(self):
+        comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment'])
+        diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator)
+
+        return diff
+
+    # Возвращает сообщения, существующие в target и source, с неэквивалентным значением и атрибутами
+    def get_target_not_equal_exist_values_with_attrs(self):
+        diff = py_.difference_with(self.target_elements, self.source_elements,
+                                   comparator=lambda a, b: a.element.equals(b.element,
+                                                                            ignored_fields=['span', 'id', 'comment']))
+        exist = py_.intersection(diff, self.get_exist_id_names(self.target_elements, self.source_elements))
+
+        return exist
+
+    def find_message_by_id_name(self, id_name, list):
+        return py_.find(list, lambda el: el.get_id_name() == id_name)