From 0174b457535207818c78f5a661b25a0f7c66e521 Mon Sep 17 00:00:00 2001 From: Zekins <136648667+Zekins3366@users.noreply.github.com> Date: Tue, 26 Mar 2024 14:49:02 +0300 Subject: [PATCH] Add files via upload --- Tools/ss14_ru/file.py | 91 ++++++++ Tools/ss14_ru/fluentast.py | 187 +++++++++++++++ Tools/ss14_ru/fluentastcomparer.py | 100 ++++++++ Tools/ss14_ru/fluentastmanager.py | 25 ++ Tools/ss14_ru/fluentformatter.py | 37 +++ Tools/ss14_ru/keyfinder.py | 220 ++++++++++++++++++ .../lokalise_fluent_ast_comparer_manager.py | 48 ++++ Tools/ss14_ru/lokalise_project.py | 33 +++ Tools/ss14_ru/lokalisemodels.py | 64 +++++ Tools/ss14_ru/project.py | 31 +++ Tools/ss14_ru/requirements.txt | Bin 0 -> 142 bytes Tools/ss14_ru/translationsassembler.py | 67 ++++++ Tools/ss14_ru/yamlextractor.py | 85 +++++++ Tools/ss14_ru/yamlmodels.py | 26 +++ 14 files changed, 1014 insertions(+) create mode 100644 Tools/ss14_ru/file.py create mode 100644 Tools/ss14_ru/fluentast.py create mode 100644 Tools/ss14_ru/fluentastcomparer.py create mode 100644 Tools/ss14_ru/fluentastmanager.py create mode 100644 Tools/ss14_ru/fluentformatter.py create mode 100644 Tools/ss14_ru/keyfinder.py create mode 100644 Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py create mode 100644 Tools/ss14_ru/lokalise_project.py create mode 100644 Tools/ss14_ru/lokalisemodels.py create mode 100644 Tools/ss14_ru/project.py create mode 100644 Tools/ss14_ru/requirements.txt create mode 100644 Tools/ss14_ru/translationsassembler.py create mode 100644 Tools/ss14_ru/yamlextractor.py create mode 100644 Tools/ss14_ru/yamlmodels.py diff --git a/Tools/ss14_ru/file.py b/Tools/ss14_ru/file.py new file mode 100644 index 00000000000..3ec0424c22f --- /dev/null +++ b/Tools/ss14_ru/file.py @@ -0,0 +1,91 @@ +import typing + +from fluent.syntax import ast +from yamlmodels import YAMLElements +import os + + +class File: + def __init__(self, full_path): + self.full_path = full_path + + def read_data(self): + file = open(self.full_path, 'r', encoding='utf8') + # replace необходим для того, чтобы 1-е сообщение не считалось ast.Junk + file_data = file.read().replace('', '') + file.close() + + return file_data + + def save_data(self, file_data: typing.AnyStr): + os.makedirs(os.path.dirname(self.full_path), exist_ok=True) + file = open(self.full_path, 'w', encoding='utf8') + file.write(file_data) + file.close() + + def get_relative_path(self, base_path): + return os.path.relpath(self.full_path, base_path) + + def get_relative_path_without_extension(self, base_path): + return self.get_relative_path(base_path).split('.', maxsplit=1)[0] + + def get_relative_parent_dir(self, base_path): + return os.path.relpath(self.get_parent_dir(), base_path) + + def get_parent_dir(self): + return os.path.dirname(self.full_path) + + def get_name(self): + return os.path.basename(self.full_path).split('.')[0] + + +class FluentFile(File): + def __init__(self, full_path): + super().__init__(full_path) + self.full_path = full_path + + def parse_data(self, file_data: typing.AnyStr): + from fluent.syntax import FluentParser + + return FluentParser().parse(file_data) + + def serialize_data(self, parsed_file_data: ast.Resource): + from fluent.syntax import FluentSerializer + + return FluentSerializer(with_junk=True).serialize(parsed_file_data) + + def read_serialized_data(self): + return self.serialize_data(self.parse_data(self.read_data())) + + def read_parsed_data(self): + return self.parse_data(self.read_data()) + + +class YAMLFluentFileAdapter(File): + def __init__(self, full_path): + super().__init__(full_path) + self.full_path = full_path + + # def create_fluent_from_yaml_elements(self, yaml_elements): + + + +class YAMLFile(File): + def __init__(self, full_path): + super().__init__(full_path) + + def parse_data(self, file_data: typing.AnyStr): + import yaml + + return yaml.load(file_data, Loader=yaml.BaseLoader) + + def get_elements(self, parsed_data): + + if isinstance(parsed_data, list): + elements = YAMLElements(parsed_data).elements + # элемент может быть None, если имеет неизвестный тип + exist_elements = list(filter(lambda el: el, elements)) + + return exist_elements + + return [] diff --git a/Tools/ss14_ru/fluentast.py b/Tools/ss14_ru/fluentast.py new file mode 100644 index 00000000000..b6f0a58f3aa --- /dev/null +++ b/Tools/ss14_ru/fluentast.py @@ -0,0 +1,187 @@ +import typing + +from fluent.syntax import ast, FluentParser, FluentSerializer +from lokalisemodels import LokaliseKey +from pydash import py_ + + +class FluentAstAbstract: + element = None + @classmethod + def get_id_name(cls, element): + if isinstance(element, ast.Junk): + return FluentAstJunk(element).get_id_name() + elif isinstance(element, ast.Message): + return FluentAstMessage(element).get_id_name() + elif isinstance(element, ast.Term): + return FluentAstTerm(element).get_id_name() + else: + return None + + @classmethod + def create_element(cls, element): + if isinstance(element, ast.Junk): + cls.element = FluentAstJunk(element) + return cls.element + elif isinstance(element, ast.Message): + cls.element = FluentAstMessage(element) + return cls.element + elif isinstance(element, ast.Term): + cls.element = FluentAstTerm(element) + return cls.element + else: + return None + + +class FluentAstMessage: + def __init__(self, message: ast.Message): + self.message = message + self.element = message + + def get_id_name(self): + return self.message.id.name + + +class FluentAstTerm: + def __init__(self, term: ast.Term): + self.term = term + self.element = term + + def get_id_name(self): + return self.term.id.name + + +class FluentAstAttribute: + def __init__(self, id, value, parent_key = None): + self.id = id + self.value = value + self.parent_key = parent_key + + +class FluentAstAttributeFactory: + @classmethod + def from_yaml_element(cls, element): + attrs = [] + if element.description: + attrs.append(FluentAstAttribute('desc', element.description)) + + if element.suffix: + attrs.append(FluentAstAttribute('suffix', element.suffix)) + + if not len(attrs): + return None + + return attrs + + +class FluentAstJunk: + def __init__(self, junk: ast.Junk): + self.junk = junk + self.element = junk + + def get_id_name(self): + return self.junk.content.split('=')[0].strip() + + +class FluentSerializedMessage: + @classmethod + def from_yaml_element(cls, id, value, attributes, parent_id = None, raw_key = False): + if not value and not id and not parent_id: + return None + + if not attributes: + attributes = [] + + if len(list(filter(lambda attr: attr.id == 'desc', attributes))) == 0: + if parent_id: + attributes.append(FluentAstAttribute('desc', '{ ' + FluentSerializedMessage.get_key(parent_id) + '.desc' + ' }')); + else: + attributes.append(FluentAstAttribute('desc', '{ "" }')) + + message = f'{cls.get_key(id, raw_key)} = {cls.get_value(value, parent_id)}\n' + + if attributes and len(attributes): + full_message = message + + for attr in attributes: + fluent_newlines = attr.value.replace("\n", "\n "); + full_message = cls.add_attr(full_message, attr.id, fluent_newlines, raw_key=raw_key) + + desc_attr = py_.find(attributes, lambda a: a.id == 'desc') + if not desc_attr and parent_id: + full_message = cls.add_attr(full_message, 'desc', '{ ' + FluentSerializedMessage.get_key(parent_id) + '.desc' + ' }') + + return full_message + + return cls.to_serialized_message(message) + + @classmethod + def from_lokalise_keys(cls, keys: typing.List[LokaliseKey]): + attributes_keys = list(filter(lambda k: k.is_attr, keys)) + attributes = list(map(lambda k: FluentAstAttribute(id='.{name}'.format(name=k.get_key_last_name(k.key_name)), + value=FluentSerializedMessage.get_attr(k, k.get_key_last_name(k.key_name)), parent_key=k.get_parent_key()), + attributes_keys)) + attributes_group = py_.group_by(attributes, 'parent_key') + + serialized_message = '' + for key in keys: + if key.is_attr: + continue + key_name = key.get_key_last_name(key.key_name) + key_value = key.get_translation('ru').data['translation'] + key_attributes = [] + + if len(attributes_group): + k = f'{key.get_key_base_name(key.key_name)}.{key_name}' + key_attributes = attributes_group[k] if k in attributes_group else [] + + message = key.serialize_message() + full_message = cls.from_yaml_element(key_name, key_value, key_attributes, key.get_parent_key(), True) + + if full_message: + serialized_message = serialized_message + '\n' + full_message + elif message: + serialized_message = serialized_message + '\n' + message + else: + raise Exception('Что-то пошло не так') + + return serialized_message + + @staticmethod + def get_attr(k, name, parent_id = None): + if parent_id: + return "{ " + parent_id + f'.{name}' + " }" + else: + return k.get_translation('ru').data['translation'] + + + @staticmethod + def to_serialized_message(string_message): + if not string_message: + return None + + ast_message = FluentParser().parse(string_message) + serialized = FluentSerializer(with_junk=True).serialize(ast_message) + + return serialized if serialized else '' + + @staticmethod + def add_attr(message_str, attr_key, attr_value, raw_key = False): + prefix = '' if raw_key else '.' + return f'{message_str}\n {prefix}{attr_key} = {attr_value}' + + @staticmethod + def get_value(value, parent_id): + if value: + return value + elif parent_id: + return '{ ' + FluentSerializedMessage.get_key(parent_id) + ' }' + else: + return '{ "" }' + + @staticmethod + def get_key(id, raw = False): + if raw: + return f'{id}' + else: + return f'ent-{id}' diff --git a/Tools/ss14_ru/fluentastcomparer.py b/Tools/ss14_ru/fluentastcomparer.py new file mode 100644 index 00000000000..d6a1b79b6b1 --- /dev/null +++ b/Tools/ss14_ru/fluentastcomparer.py @@ -0,0 +1,100 @@ +from fluent.syntax import ast +from fluentast import FluentAstAbstract +from pydash import py_ + + +class FluentAstComparer: + def __init__(self, sourse_parsed: ast.Resource, target_parsed: ast.Resource): + self.sourse_parsed = sourse_parsed + self.target_parsed = target_parsed + self.source_elements = list( + filter(lambda el: el, list(map(lambda e: FluentAstAbstract.create_element(e), sourse_parsed.body)))) + self.target_elements = list( + filter(lambda el: el, list(map(lambda e: FluentAstAbstract.create_element(e), target_parsed.body)))) + + # Возвращает полностью эквивалентные сообщения (не считая span) + def get_equal_elements(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span']) + + return py_.intersection_with(self.source_elements, self.target_elements, comparator=comparator) + + # Возвращает полностью неэквивалентные сообщения (не считая span) + def get_not_equal_elements(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span']) + diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator) + + return diff + + # Возвращает сообщения с эквивалентными именами ключей + def get_equal_id_names(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes']) + eq = py_.intersection_with(self.source_elements, self.target_elements, comparator=comparator) + + return eq + + # Возвращает сообщения с неэквивалентными именами ключей + def get_not_equal_id_names(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes']) + diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator) + + return diff + + # Возвращает сообщения target, существующие в source + def get_exist_id_names(self, source, target): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes']) + eq = py_.intersection_with(source, target, comparator=comparator) + + return eq + + # Возвращает сообщения target, существующие в source + def get_not_exist_id_names(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'value', 'comment', 'attributes']) + diff = py_.difference_with(self.target_elements, self.source_elements, comparator=comparator) + + return diff + + # Возвращает сообщения с эквивалентным значением и атрибутами + def get_equal_values_with_attrs(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment']) + eq = py_.intersection_with(self.target_elements, self.source_elements, comparator=comparator) + + return eq + + # Возвращает сообщения из source с неэквивалентным значением и атрибутами + def get_not_equal_values_with_attrs(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment']) + diff = py_.difference_with(self.source_elements, self.target_elements, + comparator=lambda a, b: a.element.equals(b.element, + ignored_fields=['span', 'id', 'comment'])) + + return diff + + # Возвращает сообщения из source, существующие в target и source, с неэквивалентным значением и атрибутами + def get_not_equal_exist_values_with_attrs(self): + diff = py_.difference_with(self.source_elements, self.target_elements, + comparator=lambda a, b: a.element.equals(b.element, + ignored_fields=['span', 'id', 'comment'])) + ex = self.get_exist_id_names(self.source_elements, self.target_elements) + exist = py_.intersection(diff, ex) + + return exist + + # Возвращает сообщения из target с неэквивалентным значением и атрибутами + + def get_target_not_equal_values_with_attrs(self): + comparator = lambda a, b: a.element.equals(b.element, ignored_fields=['span', 'id', 'comment']) + diff = py_.difference_with(self.source_elements, self.target_elements, comparator=comparator) + + return diff + + # Возвращает сообщения, существующие в target и source, с неэквивалентным значением и атрибутами + def get_target_not_equal_exist_values_with_attrs(self): + diff = py_.difference_with(self.target_elements, self.source_elements, + comparator=lambda a, b: a.element.equals(b.element, + ignored_fields=['span', 'id', 'comment'])) + exist = py_.intersection(diff, self.get_exist_id_names(self.target_elements, self.source_elements)) + + return exist + + def find_message_by_id_name(self, id_name, list): + return py_.find(list, lambda el: el.get_id_name() == id_name) diff --git a/Tools/ss14_ru/fluentastmanager.py b/Tools/ss14_ru/fluentastmanager.py new file mode 100644 index 00000000000..8bc420827c3 --- /dev/null +++ b/Tools/ss14_ru/fluentastmanager.py @@ -0,0 +1,25 @@ +from fluent.syntax import ast +from fluentast import FluentAstAbstract + + +class FluentAstManager: + def __init__(self, sourse_parsed: ast.Resource, target_parsed: ast.Resource): + self.sourse_parsed = sourse_parsed + self.target_parsed = target_parsed + self.source_elements = list(map(lambda e: FluentAstAbstract.create_element(e), sourse_parsed.body)) + self.target_elements = list(map(lambda e: FluentAstAbstract.create_element(e), target_parsed.body)) + + def update_by_index(self, index, update_element: ast.Message): + source_element = None + + try: + source_element = self.sourse_parsed.body[index] + except: + raise Exception(f'Нет элемента с индексом {index}') + + if not source_element: + raise Exception(f'Элемен с индексом {index} не существует') + + self.sourse_parsed.body[index] = update_element + + return self.sourse_parsed diff --git a/Tools/ss14_ru/fluentformatter.py b/Tools/ss14_ru/fluentformatter.py new file mode 100644 index 00000000000..0937b4146c2 --- /dev/null +++ b/Tools/ss14_ru/fluentformatter.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +# Форматтер, приводящий fluent-файлы (.ftl) в соответствие стайлгайду +# path - путь к папке, содержащий форматируемые файлы. Для форматирования всего проекта, необходимо заменить значение на root_dir_path +import typing + +from file import FluentFile +from project import Project +from fluent.syntax import ast, FluentParser, FluentSerializer + + +######################################### Class defifitions ############################################################ + +class FluentFormatter: + @classmethod + def format(cls, fluent_files: typing.List[FluentFile]): + for file in fluent_files: + file_data = file.read_data() + parsed_file_data = file.parse_data(file_data) + serialized_file_data = file.serialize_data(parsed_file_data) + file.save_data(serialized_file_data) + + @classmethod + def format_serialized_file_data(cls, file_data: typing.AnyStr): + parsed_data = FluentParser().parse(file_data) + + return FluentSerializer(with_junk=True).serialize(parsed_data) + + + +######################################## Var definitions ############################################################### +project = Project() +fluent_files = project.get_fluent_files_by_dir(project.ru_locale_dir_path) + +######################################################################################################################## + +FluentFormatter.format(fluent_files) diff --git a/Tools/ss14_ru/keyfinder.py b/Tools/ss14_ru/keyfinder.py new file mode 100644 index 00000000000..079d884024d --- /dev/null +++ b/Tools/ss14_ru/keyfinder.py @@ -0,0 +1,220 @@ +import typing +import logging + +from pydash import py_ + +from file import FluentFile +from fluentast import FluentAstAbstract +from fluentformatter import FluentFormatter +from project import Project +from fluent.syntax import ast, FluentParser, FluentSerializer + + +# Осуществляет актуализацию ключей. Находит файлы английского перевода, проверяет: есть ли русскоязычная пара +# Если нет - создаёт файл с копией переводов из англоязычного +# Далее, пофайлово проверяются ключи. Если в английском файле больше ключей - создает недостающие в русском, с английской копией перевода +# Отмечает русские файлы, в которых есть те ключи, что нет в аналогичных английских +# Отмечает русские файлы, у которых нет англоязычной пары + +######################################### Class defifitions ############################################################ +class RelativeFile: + def __init__(self, file: FluentFile, locale: typing.AnyStr, relative_path_from_locale: typing.AnyStr): + self.file = file + self.locale = locale + self.relative_path_from_locale = relative_path_from_locale + + +class FilesFinder: + def __init__(self, project: Project): + self.project: Project = project + self.created_files: typing.List[FluentFile] = [] + + def get_relative_path_dict(self, file: FluentFile, locale): + if locale == 'ru-RU': + return RelativeFile(file=file, locale=locale, + relative_path_from_locale=file.get_relative_path(self.project.ru_locale_dir_path)) + elif locale == 'en-US': + return RelativeFile(file=file, locale=locale, + relative_path_from_locale=file.get_relative_path(self.project.en_locale_dir_path)) + else: + raise Exception(f'Локаль {locale} не поддерживается') + + def get_file_pair(self, en_file: FluentFile) -> typing.Tuple[FluentFile, FluentFile]: + ru_file_path = en_file.full_path.replace('en-US', 'ru-RU') + ru_file = FluentFile(ru_file_path) + + return en_file, ru_file + + def execute(self): + self.created_files = [] + groups = self.get_files_pars() + keys_without_pair = list(filter(lambda g: len(groups[g]) < 2, groups)) + + for key_without_pair in keys_without_pair: + relative_file: RelativeFile = groups.get(key_without_pair)[0] + + if relative_file.locale == 'en-US': + ru_file = self.create_ru_analog(relative_file) + self.created_files.append(ru_file) + elif relative_file.locale == 'ru-RU': + is_engine_files = "robust-toolbox" in (relative_file.file.full_path) + is_corvax_files = "corvax" in (relative_file.file.full_path) + if not is_engine_files and not is_corvax_files: + self.warn_en_analog_not_exist(relative_file) + else: + raise Exception(f'Файл {relative_file.file.full_path} имеет неизвестную локаль "{relative_file.locale}"') + + return self.created_files + + def get_files_pars(self): + en_fluent_files = self.project.get_fluent_files_by_dir(project.en_locale_dir_path) + ru_fluent_files = self.project.get_fluent_files_by_dir(project.ru_locale_dir_path) + + en_fluent_relative_files = list(map(lambda f: self.get_relative_path_dict(f, 'en-US'), en_fluent_files)) + ru_fluent_relative_files = list(map(lambda f: self.get_relative_path_dict(f, 'ru-RU'), ru_fluent_files)) + relative_files = py_.flatten_depth(py_.concat(en_fluent_relative_files, ru_fluent_relative_files), depth=1) + + return py_.group_by(relative_files, 'relative_path_from_locale') + + def create_ru_analog(self, en_relative_file: RelativeFile) -> FluentFile: + en_file: FluentFile = en_relative_file.file + en_file_data = en_file.read_data() + ru_file_path = en_file.full_path.replace('en-US', 'ru-RU') + ru_file = FluentFile(ru_file_path) + ru_file.save_data(en_file_data) + + logging.info(f'Создан файл {ru_file_path} с переводами из английского файла') + + return ru_file + + def warn_en_analog_not_exist(self, ru_relative_file: RelativeFile): + file: FluentFile = ru_relative_file.file + en_file_path = file.full_path.replace('ru-RU', 'en-US') + + logging.warning(f'Файл {file.full_path} не имеет английского аналога по пути {en_file_path}') + + +class KeyFinder: + def __init__(self, files_dict): + self.files_dict = files_dict + self.changed_files: typing.List[FluentFile] = [] + + def execute(self) -> typing.List[FluentFile]: + self.changed_files = [] + for pair in self.files_dict: + ru_relative_file = py_.find(self.files_dict[pair], {'locale': 'ru-RU'}) + en_relative_file = py_.find(self.files_dict[pair], {'locale': 'en-US'}) + + if not en_relative_file or not ru_relative_file: + continue + + ru_file: FluentFile = ru_relative_file.file + en_file: FluentFile = en_relative_file.file + + self.compare_files(en_file, ru_file) + + return self.changed_files + + + def compare_files(self, en_file, ru_file): + ru_file_parsed: ast.Resource = ru_file.parse_data(ru_file.read_data()) + en_file_parsed: ast.Resource = en_file.parse_data(en_file.read_data()) + + self.write_to_ru_files(ru_file, ru_file_parsed, en_file_parsed) + self.log_not_exist_en_files(en_file, ru_file_parsed, en_file_parsed) + + + def write_to_ru_files(self, ru_file, ru_file_parsed, en_file_parsed): + for idx, en_message in enumerate(en_file_parsed.body): + if isinstance(en_message, ast.ResourceComment) or isinstance(en_message, ast.GroupComment) or isinstance(en_message, ast.Comment): + continue + + ru_message_analog_idx = py_.find_index(ru_file_parsed.body, lambda ru_message: self.find_duplicate_message_id_name(ru_message, en_message)) + have_changes = False + + # Attributes + if getattr(en_message, 'attributes', None) and ru_message_analog_idx != -1: + if not ru_file_parsed.body[ru_message_analog_idx].attributes: + ru_file_parsed.body[ru_message_analog_idx].attributes = en_message.attributes + have_changes = True + else: + for en_attr in en_message.attributes: + ru_attr_analog = py_.find(ru_file_parsed.body[ru_message_analog_idx].attributes, lambda ru_attr: ru_attr.id.name == en_attr.id.name) + if not ru_attr_analog: + ru_file_parsed.body[ru_message_analog_idx].attributes.append(en_attr) + have_changes = True + + # New elements + if ru_message_analog_idx == -1: + ru_file_body = ru_file_parsed.body + if (len(ru_file_body) >= idx + 1): + ru_file_parsed = self.append_message(ru_file_parsed, en_message, idx) + else: + ru_file_parsed = self.push_message(ru_file_parsed, en_message) + have_changes = True + + if have_changes: + serialized = serializer.serialize(ru_file_parsed) + self.save_and_log_file(ru_file, serialized, en_message) + + def log_not_exist_en_files(self, en_file, ru_file_parsed, en_file_parsed): + for idx, ru_message in enumerate(ru_file_parsed.body): + if isinstance(ru_message, ast.ResourceComment) or isinstance(ru_message, ast.GroupComment) or isinstance(ru_message, ast.Comment): + continue + + en_message_analog = py_.find(en_file_parsed.body, lambda en_message: self.find_duplicate_message_id_name(ru_message, en_message)) + + if not en_message_analog: + logging.warning(f'Ключ "{FluentAstAbstract.get_id_name(ru_message)}" не имеет английского аналога по пути {en_file.full_path}"') + + def append_message(self, ru_file_parsed, en_message, en_message_idx): + ru_message_part_1 = ru_file_parsed.body[0:en_message_idx] + ru_message_part_middle = [en_message] + ru_message_part_2 = ru_file_parsed.body[en_message_idx:] + new_body = py_.flatten_depth([ru_message_part_1, ru_message_part_middle, ru_message_part_2], depth=1) + ru_file_parsed.body = new_body + + return ru_file_parsed + + def push_message(self, ru_file_parsed, en_message): + ru_file_parsed.body.append(en_message) + return ru_file_parsed + + def save_and_log_file(self, file, file_data, message): + file.save_data(file_data) + logging.info(f'В файл {file.full_path} добавлен ключ "{FluentAstAbstract.get_id_name(message)}"') + self.changed_files.append(file) + + def find_duplicate_message_id_name(self, ru_message, en_message): + ru_element_id_name = FluentAstAbstract.get_id_name(ru_message) + en_element_id_name = FluentAstAbstract.get_id_name(en_message) + + if not ru_element_id_name or not en_element_id_name: + return False + + if ru_element_id_name == en_element_id_name: + return ru_message + else: + return None + +######################################## Var definitions ############################################################### + +logging.basicConfig(level = logging.INFO) +project = Project() +parser = FluentParser() +serializer = FluentSerializer(with_junk=True) +files_finder = FilesFinder(project) +key_finder = KeyFinder(files_finder.get_files_pars()) + +######################################################################################################################## + +print('Проверка актуальности файлов ...') +created_files = files_finder.execute() +if len(created_files): + print('Форматирование созданных файлов ...') + FluentFormatter.format(created_files) +print('Проверка актуальности ключей ...') +changed_files = key_finder.execute() +if len(changed_files): + print('Форматирование изменённых файлов ...') + FluentFormatter.format(changed_files) diff --git a/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py b/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py new file mode 100644 index 00000000000..4eff73e3b50 --- /dev/null +++ b/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py @@ -0,0 +1,48 @@ +from fluent.syntax import ast + +from fluentast import FluentAstMessage +from fluentastcomparer import FluentAstComparer +from fluentastmanager import FluentAstManager + + +class LokaliseFluentAstComparerManager: + def __init__(self, sourse_parsed: ast.Resource, target_parsed: ast.Resource): + self.sourse_parsed = sourse_parsed + self.target_parsed = target_parsed + self.comparer = FluentAstComparer(sourse_parsed, target_parsed) + self.ast_manager = FluentAstManager(sourse_parsed, target_parsed) + + def for_update(self): + for_update = self.comparer.get_not_equal_exist_values_with_attrs() + + if not len(for_update): + return [] + + return for_update + + def update(self, for_update): + for update in for_update: + idx = self.comparer.sourse_parsed.body.index(update.element) + update_mess: FluentAstMessage = self.comparer.find_message_by_id_name(update.get_id_name(), + self.comparer.target_elements) + self.ast_manager.update_by_index(idx, update_mess.element) + + return self.ast_manager.sourse_parsed + + def for_delete(self): + for_delete = self.comparer.get_not_exist_id_names() + + if len(for_delete): + keys = list(map(lambda el: el.get_id_name(), for_delete)) + print(f'Следующие ключи есть в lokalise, но нет в файле. Возможно, их нужно удалить из lokalise: {keys}') + + return for_delete + + def for_create(self): + for_create = self.comparer.get_not_equal_id_names() + + if len(for_create): + keys = list(map(lambda el: el.get_id_name(), for_create)) + print(f'Следующих ключей файла нет в lokalise. Необходимо добавить: {keys}') + + return for_create diff --git a/Tools/ss14_ru/lokalise_project.py b/Tools/ss14_ru/lokalise_project.py new file mode 100644 index 00000000000..b16a60a6d35 --- /dev/null +++ b/Tools/ss14_ru/lokalise_project.py @@ -0,0 +1,33 @@ +import lokalise +import typing +from lokalisemodels import LokaliseKey +from pydash import py_ + +class LokaliseProject: + def __init__(self, project_id, personal_token): + self.project_id = project_id + self.personal_token = personal_token + self.client = lokalise.Client(self.personal_token) + + def get_all_keys(self) -> typing.List[LokaliseKey]: + page = 1 + keys = self.get_keys(page=page) + keys_items: typing.List[lokalise.client.KeyModel] = [] + general_count = 0 + + while (general_count < keys.total_count): + general_count = general_count + len(keys.items) + keys_items = py_.flatten_depth(py_.concat(keys_items, keys.items), depth=1) + + if (general_count == keys.total_count): + break + + next_page = page = page + 1 + keys = self.get_keys(page=next_page) + + sorted_list = py_.sort(keys_items, key=lambda item: item.translations_modified_at_timestamp, reverse=True) + + return list(map(lambda k: LokaliseKey(k), sorted_list)) + + def get_keys(self, page): + return self.client.keys(self.project_id, {'page': page, 'limit': 5000, 'include_translations': 1}) diff --git a/Tools/ss14_ru/lokalisemodels.py b/Tools/ss14_ru/lokalisemodels.py new file mode 100644 index 00000000000..66a925f105e --- /dev/null +++ b/Tools/ss14_ru/lokalisemodels.py @@ -0,0 +1,64 @@ +import typing +import os +from pydash import py_ +from project import Project + +class LocalePath: + def __init__(self, relative_file_path): + self.ru = os.path.join(Project().ru_locale_dir_path, relative_file_path) + self.en = os.path.join(Project().en_locale_dir_path, relative_file_path) + + +class LokaliseTranslation: + def __init__(self, data, key_name: typing.AnyStr): + self.key_name = key_name, + self.data = data + +class LokaliseKey: + def __init__(self, data): + self.data = data + self.key_name = self.data.key_name['web'] + self.key_base_name = self.get_key_base_name(self.key_name) + self.is_attr = self.check_is_attr() + + def get_file_path(self): + + relative_dir_path = '{relative_file_path}.ftl'.format( + relative_file_path='/'.join(self.data.key_name['web'].split('.')[0].split('::'))) + + return LocalePath(relative_dir_path) + + def get_key_base_name(self, key_name): + splitted_name = key_name.split('.') + return splitted_name[0] + + def get_key_last_name(self, key_name): + splitted_name = key_name.split('.') + return py_.last(splitted_name) + + def get_parent_key(self): + if self.is_attr: + splitted_name = self.key_name.split('.')[0:-1] + return '.'.join(splitted_name) + else: + return None + + def check_is_attr(self): + return len(self.key_name.split('.')) > 2 + + def serialize(self): + if self.is_attr: + return self.serialize_attr() + else: + return self.serialize_message() + + + + def serialize_attr(self): + return '.{name} = {value}'.format(name=self.get_key_last_name(self.key_name), value=self.get_translation('ru').data['translation']) + + def serialize_message(self): + return '{name} = {value}'.format(name=self.get_key_last_name(self.key_name), value=self.get_translation('ru').data['translation']) + + def get_translation(self, language_iso='ru'): + return list(map(lambda data: LokaliseTranslation(key_name=self.data.key_name['web'], data=data), py_.filter(self.data.translations, {'language_iso': language_iso})))[0] diff --git a/Tools/ss14_ru/project.py b/Tools/ss14_ru/project.py new file mode 100644 index 00000000000..2bf90e0790e --- /dev/null +++ b/Tools/ss14_ru/project.py @@ -0,0 +1,31 @@ +import pathlib +import os +import glob +from file import FluentFile + +class Project: + def __init__(self): + self.base_dir_path = pathlib.Path(os.path.abspath(os.curdir)).parent.parent.resolve() + self.resources_dir_path = os.path.join(self.base_dir_path, 'Resources') + self.locales_dir_path = os.path.join(self.resources_dir_path, 'Locale') + self.ru_locale_dir_path = os.path.join(self.locales_dir_path, 'ru-RU') + self.en_locale_dir_path = os.path.join(self.locales_dir_path, 'en-US') + self.prototypes_dir_path = os.path.join(self.resources_dir_path, "Prototypes") + self.en_locale_prototypes_dir_path = os.path.join(self.en_locale_dir_path, 'ss14-ru', 'prototypes') + self.ru_locale_prototypes_dir_path = os.path.join(self.ru_locale_dir_path, 'ss14-ru', 'prototypes') + + def get_files_paths_by_dir(self, dir_path, files_extenstion): + return glob.glob(f'{dir_path}/**/*.{files_extenstion}', recursive=True) + + def get_fluent_files_by_dir(self, dir_path): + files = [] + files_paths_list = glob.glob(f'{dir_path}/**/*.ftl', recursive=True) + + for file_path in files_paths_list: + try: + files.append(FluentFile(file_path)) + except: + continue + + return files + diff --git a/Tools/ss14_ru/requirements.txt b/Tools/ss14_ru/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..c68858f01804994cdda64a0739e37adf7324799a GIT binary patch literal 142 zcmezWFO4CGp_Cz&A&;Si!4?P&81xtn!K48LF9R1t0YfE23PU18F+&De-W(_oQezC3 z4FJkUGB`5$GWdXH&A_rCeI-Dd0)|YWx#