From a4f8d77fa9143a24882ef8ab96a0dd02aa9e0f6e Mon Sep 17 00:00:00 2001 From: Evgencheg <7064926@gmail.com> Date: Sun, 6 Oct 2024 15:25:28 +0300 Subject: [PATCH 1/6] Fix locale scripts --- Tools/ss14_ru/__init__.py | 2 +- Tools/ss14_ru/clean_duplicates.py | 119 ++++++++++++++++++ Tools/ss14_ru/clean_empty.py | 61 +++++++++ Tools/ss14_ru/file.py | 43 ++++++- Tools/ss14_ru/fluentast.py | 2 +- Tools/ss14_ru/fluentastcomparer.py | 2 +- Tools/ss14_ru/fluentastmanager.py | 2 +- Tools/ss14_ru/fluentformatter.py | 2 +- Tools/ss14_ru/keyfinder.py | 2 +- .../lokalise_fluent_ast_comparer_manager.py | 2 +- Tools/ss14_ru/lokalise_project.py | 2 +- Tools/ss14_ru/lokalisemodels.py | 2 +- Tools/ss14_ru/project.py | 2 +- Tools/ss14_ru/requirements.txt | Bin 142 -> 146 bytes Tools/ss14_ru/translation.bat | 7 ++ Tools/ss14_ru/translation.sh | 12 ++ Tools/ss14_ru/translationsassembler.py | 2 +- Tools/ss14_ru/yamlextractor.py | 22 +++- Tools/ss14_ru/yamlmodels.py | 2 +- 19 files changed, 268 insertions(+), 20 deletions(-) create mode 100644 Tools/ss14_ru/clean_duplicates.py create mode 100644 Tools/ss14_ru/clean_empty.py create mode 100644 Tools/ss14_ru/translation.bat create mode 100644 Tools/ss14_ru/translation.sh diff --git a/Tools/ss14_ru/__init__.py b/Tools/ss14_ru/__init__.py index 8fc809dcf53..c7fe81101f3 100644 --- a/Tools/ss14_ru/__init__.py +++ b/Tools/ss14_ru/__init__.py @@ -1 +1 @@ -from fluentformatter import FluentFile, FluentFormatter +from fluentformatter import FluentFile, FluentFormatter diff --git a/Tools/ss14_ru/clean_duplicates.py b/Tools/ss14_ru/clean_duplicates.py new file mode 100644 index 00000000000..daea65b4bc9 --- /dev/null +++ b/Tools/ss14_ru/clean_duplicates.py @@ -0,0 +1,119 @@ +import os +import re +import chardet +from datetime import datetime + +def find_top_level_dir(start_dir): + marker_file = 'SpaceStation14.sln' + current_dir = start_dir + while True: + if marker_file in os.listdir(current_dir): + return current_dir + parent_dir = os.path.dirname(current_dir) + if parent_dir == current_dir: + print(f"Не удалось найти {marker_file} начиная с {start_dir}") + exit(-1) + current_dir = parent_dir +def find_ftl_files(root_dir): + ftl_files = [] + for root, dirs, files in os.walk(root_dir): + for file in files: + if file.endswith('.ftl'): + ftl_files.append(os.path.join(root, file)) + return ftl_files + +def detect_encoding(file_path): + with open(file_path, 'rb') as file: + raw_data = file.read() + return chardet.detect(raw_data)['encoding'] + +def parse_ent_blocks(file_path): + try: + encoding = detect_encoding(file_path) + with open(file_path, 'r', encoding=encoding) as file: + content = file.read() + except UnicodeDecodeError: + print(f"Ошибка при чтении файла {file_path}. Попытка чтения в UTF-8.") + try: + with open(file_path, 'r', encoding='utf-8') as file: + content = file.read() + except UnicodeDecodeError: + print(f"Не удалось прочитать файл {file_path}. Пропускаем.") + return {} + + ent_blocks = {} + current_ent = None + current_block = [] + + for line in content.split('\n'): + if line.startswith('ent-'): + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + current_ent = line.split('=')[0].strip() + current_block = [line] + elif current_ent and (line.strip().startswith('.desc') or line.strip().startswith('.suffix')): + current_block.append(line) + elif line.strip() == '': + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + current_ent = None + current_block = [] + else: + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + current_ent = None + current_block = [] + + if current_ent: + ent_blocks[current_ent] = '\n'.join(current_block) + + return ent_blocks + +def remove_duplicates(root_dir): + ftl_files = find_ftl_files(root_dir) + all_ents = {} + removed_duplicates = [] + + for file_path in ftl_files: + ent_blocks = parse_ent_blocks(file_path) + for ent, block in ent_blocks.items(): + if ent not in all_ents: + all_ents[ent] = (file_path, block) + + for file_path in ftl_files: + try: + encoding = detect_encoding(file_path) + with open(file_path, 'r', encoding=encoding) as file: + content = file.read() + + ent_blocks = parse_ent_blocks(file_path) + for ent, block in ent_blocks.items(): + if all_ents[ent][0] != file_path: + content = content.replace(block, '') + removed_duplicates.append((ent, file_path, block)) + + content = re.sub(r'\n{3,}', '\n\n', content) + + with open(file_path, 'w', encoding=encoding) as file: + file.write(content) + except Exception as e: + print(f"Ошибка при обработке файла {file_path}: {str(e)}") + + # Сохранение лога удаленных дубликатов + log_filename = f"removed_duplicates_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + with open(log_filename, 'w', encoding='utf-8') as log_file: + for ent, file_path, block in removed_duplicates: + log_file.write(f"Удален дубликат: {ent}\n") + log_file.write(f"Файл: {file_path}\n") + log_file.write("Содержимое:\n") + log_file.write(block) + log_file.write("\n\n") + + print(f"Обработка завершена. Проверено файлов: {len(ftl_files)}") + print(f"Лог удаленных дубликатов сохранен в файл: {log_filename}") + +if __name__ == "__main__": + script_dir = os.path.dirname(os.path.abspath(__file__)) + main_folder = find_top_level_dir(script_dir) + root_dir = os.path.join(main_folder, "Resources\\Locale\\ru-RU") + remove_duplicates(root_dir) diff --git a/Tools/ss14_ru/clean_empty.py b/Tools/ss14_ru/clean_empty.py new file mode 100644 index 00000000000..d6f2efd7c30 --- /dev/null +++ b/Tools/ss14_ru/clean_empty.py @@ -0,0 +1,61 @@ +import os +import logging +from datetime import datetime + +def find_top_level_dir(start_dir): + marker_file = 'SpaceStation14.sln' + current_dir = start_dir + while True: + if marker_file in os.listdir(current_dir): + return current_dir + parent_dir = os.path.dirname(current_dir) + if parent_dir == current_dir: + print(f"Не удалось найти {marker_file} начиная с {start_dir}") + exit(-1) + current_dir = parent_dir +def setup_logging(): + log_filename = f"cleanup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + logging.basicConfig(filename=log_filename, level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s') + console = logging.StreamHandler() + console.setLevel(logging.INFO) + logging.getLogger('').addHandler(console) + return log_filename + +def remove_empty_files_and_folders(path): + removed_files = 0 + removed_folders = 0 + + for root, dirs, files in os.walk(path, topdown=False): + # Удаление пустых файлов + for file in files: + file_path = os.path.join(root, file) + if os.path.getsize(file_path) == 0: + try: + os.remove(file_path) + logging.info(f"Удален пустой файл: {file_path}") + removed_files += 1 + except Exception as e: + logging.error(f"Ошибка при удалении файла {file_path}: {str(e)}") + + # Удаление пустых папок + if not os.listdir(root): + try: + os.rmdir(root) + logging.info(f"Удалена пустая папка: {root}") + removed_folders += 1 + except Exception as e: + logging.error(f"Ошибка при удалении папки {root}: {str(e)}") + + return removed_files, removed_folders + +if __name__ == "__main__": + script_dir = os.path.dirname(os.path.abspath(__file__)) + main_folder = find_top_level_dir(script_dir) + root_dir = os.path.join(main_folder, "Resources\\Locale") + log_file = setup_logging() + + logging.info(f"Начало очистки в директории: {root_dir}") + files_removed, folders_removed = remove_empty_files_and_folders(root_dir) + logging.info(f"Очистка завершена. Удалено файлов: {files_removed}, удалено папок: {folders_removed}") + print(f"Лог операций сохранен в файл: {log_file}") diff --git a/Tools/ss14_ru/file.py b/Tools/ss14_ru/file.py index bbb53c79b1c..abaf8e1d730 100644 --- a/Tools/ss14_ru/file.py +++ b/Tools/ss14_ru/file.py @@ -1,8 +1,9 @@ -import typing +import typing from fluent.syntax import ast from yamlmodels import YAMLElements import os +import re class File: @@ -42,17 +43,53 @@ def get_name(self): class FluentFile(File): def __init__(self, full_path): super().__init__(full_path) + + self.newline_exceptions_regex = re.compile(r"^\s*[\[\]{}#%^*]") + self.newline_remover_tag = "%ERASE_NEWLINE%" + self.newline_remover_regex = re.compile(r"\n?\s*" + self.newline_remover_tag) + + "%ERASE_NEWLINE%" + self.full_path = full_path + def kludge(self, element): + return str.replace( + element.value, + self.prefixed_newline, + self.prefixed_newline_substitute + ) + + def parse_data(self, file_data: typing.AnyStr): from fluent.syntax import FluentParser - return FluentParser().parse(file_data) + parsed_data = FluentParser().parse(file_data) + + for body_element in parsed_data.body: + if not isinstance(body_element, ast.Term) and not isinstance(body_element, ast.Message): + continue + + if not len(body_element.value.elements): + continue + + first_element = body_element.value.elements[0] + if not isinstance(first_element, ast.TextElement): + continue + + if not self.newline_exceptions_regex.match(first_element.value): + continue + + first_element.value = f"{self.newline_remover_tag}{first_element.value}" + + return parsed_data def serialize_data(self, parsed_file_data: ast.Resource): from fluent.syntax import FluentSerializer - return FluentSerializer(with_junk=True).serialize(parsed_file_data) + serialized_data = FluentSerializer(with_junk=True).serialize(parsed_file_data) + serialized_data = self.newline_remover_regex.sub(' ', serialized_data) + + return serialized_data def read_serialized_data(self): return self.serialize_data(self.parse_data(self.read_data())) diff --git a/Tools/ss14_ru/fluentast.py b/Tools/ss14_ru/fluentast.py index 20576b47051..ac2597f7667 100644 --- a/Tools/ss14_ru/fluentast.py +++ b/Tools/ss14_ru/fluentast.py @@ -1,4 +1,4 @@ -import typing +import typing from fluent.syntax import ast, FluentParser, FluentSerializer from lokalisemodels import LokaliseKey diff --git a/Tools/ss14_ru/fluentastcomparer.py b/Tools/ss14_ru/fluentastcomparer.py index 21fa9701757..0b4c49b3417 100644 --- a/Tools/ss14_ru/fluentastcomparer.py +++ b/Tools/ss14_ru/fluentastcomparer.py @@ -1,4 +1,4 @@ -from fluent.syntax import ast +from fluent.syntax import ast from fluentast import FluentAstAbstract from pydash import py_ diff --git a/Tools/ss14_ru/fluentastmanager.py b/Tools/ss14_ru/fluentastmanager.py index 76cf4f7bcdf..8ae13154d1e 100644 --- a/Tools/ss14_ru/fluentastmanager.py +++ b/Tools/ss14_ru/fluentastmanager.py @@ -1,4 +1,4 @@ -from fluent.syntax import ast +from fluent.syntax import ast from fluentast import FluentAstAbstract diff --git a/Tools/ss14_ru/fluentformatter.py b/Tools/ss14_ru/fluentformatter.py index f7a96dbb6c9..dd043b0e5df 100644 --- a/Tools/ss14_ru/fluentformatter.py +++ b/Tools/ss14_ru/fluentformatter.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python3 # Форматтер, приводящий fluent-файлы (.ftl) в соответствие стайлгайду # path - путь к папке, содержащий форматируемые файлы. Для форматирования всего проекта, необходимо заменить значение на root_dir_path diff --git a/Tools/ss14_ru/keyfinder.py b/Tools/ss14_ru/keyfinder.py index b06d4282868..317cec76dc3 100644 --- a/Tools/ss14_ru/keyfinder.py +++ b/Tools/ss14_ru/keyfinder.py @@ -1,4 +1,4 @@ -import typing +import typing import logging from pydash import py_ diff --git a/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py b/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py index cdbc84d5bd8..86e39aba45c 100644 --- a/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py +++ b/Tools/ss14_ru/lokalise_fluent_ast_comparer_manager.py @@ -1,4 +1,4 @@ -from fluent.syntax import ast +from fluent.syntax import ast from fluentast import FluentAstMessage from fluentastcomparer import FluentAstComparer diff --git a/Tools/ss14_ru/lokalise_project.py b/Tools/ss14_ru/lokalise_project.py index 05a4f4d285e..bed995793a1 100644 --- a/Tools/ss14_ru/lokalise_project.py +++ b/Tools/ss14_ru/lokalise_project.py @@ -1,4 +1,4 @@ -import lokalise +import lokalise import typing from lokalisemodels import LokaliseKey from pydash import py_ diff --git a/Tools/ss14_ru/lokalisemodels.py b/Tools/ss14_ru/lokalisemodels.py index 98e238cc18c..44846cd94ac 100644 --- a/Tools/ss14_ru/lokalisemodels.py +++ b/Tools/ss14_ru/lokalisemodels.py @@ -1,4 +1,4 @@ -import typing +import typing import os from pydash import py_ from project import Project diff --git a/Tools/ss14_ru/project.py b/Tools/ss14_ru/project.py index 7fcc9c7c329..0b4f67f4abf 100644 --- a/Tools/ss14_ru/project.py +++ b/Tools/ss14_ru/project.py @@ -1,4 +1,4 @@ -import pathlib +import pathlib import os import glob from file import FluentFile diff --git a/Tools/ss14_ru/requirements.txt b/Tools/ss14_ru/requirements.txt index c68858f01804994cdda64a0739e37adf7324799a..bb76cba1b5ec9319e7664e7a15330c80b1b49758 100644 GIT binary patch delta 37 lcmeBUoWwZ6h|_{WkHG+l4JTSlGlQs5eJ)F|7%u}C0|1C*1^EB~ delta 33 jcmbQl*vB}*h|`=wkHG+ljVD@5PmI#zG6nN_8Mqh#b2tT< diff --git a/Tools/ss14_ru/translation.bat b/Tools/ss14_ru/translation.bat new file mode 100644 index 00000000000..9b614730c6d --- /dev/null +++ b/Tools/ss14_ru/translation.bat @@ -0,0 +1,7 @@ +@echo off + +call pip install -r requirements.txt +call python ./yamlextractor.py +call python ./keyfinder.py +call python ./clean_duplicates.py +call python ./clean_empty.py diff --git a/Tools/ss14_ru/translation.sh b/Tools/ss14_ru/translation.sh new file mode 100644 index 00000000000..22affc3099d --- /dev/null +++ b/Tools/ss14_ru/translation.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env sh + +# make sure to start from script dir +if [ "$(dirname $0)" != "." ]; then + cd "$(dirname $0)" +fi + +pip install -r requirements.txt +python3 ./yamlextractor.py +python3 ./keyfinder.py +python3 ./clean_duplicates.py +python3 ./clean_empty.py diff --git a/Tools/ss14_ru/translationsassembler.py b/Tools/ss14_ru/translationsassembler.py index 4ce893626f8..930e9a103b6 100644 --- a/Tools/ss14_ru/translationsassembler.py +++ b/Tools/ss14_ru/translationsassembler.py @@ -1,4 +1,4 @@ -import logging +import logging import typing from fluent.syntax import FluentParser, FluentSerializer diff --git a/Tools/ss14_ru/yamlextractor.py b/Tools/ss14_ru/yamlextractor.py index aa8511263e8..2891402018e 100644 --- a/Tools/ss14_ru/yamlextractor.py +++ b/Tools/ss14_ru/yamlextractor.py @@ -1,4 +1,4 @@ -import os +import os from fluent.syntax.parser import FluentParser from fluent.syntax.serializer import FluentSerializer @@ -34,10 +34,23 @@ def execute(self): en_fluent_file_path = self.create_en_fluent_file(relative_parent_dir, file_name, pretty_fluent_file_serialized) ru_fluent_file_path = self.create_ru_fluent_file(en_fluent_file_path) + @classmethod + def serialize_yaml_element(cls, element): + parent_id = element.parent_id + if isinstance(parent_id, list): + parent_id = parent_id[0] if parent_id else 'None' + + message = FluentSerializedMessage.from_yaml_element( + element.id, element.name, + FluentAstAttributeFactory.from_yaml_element(element), + parent_id + ) + + return message + + def get_serialized_fluent_from_yaml_elements(self, yaml_elements): - fluent_serialized_messages = list( - map(lambda el: FluentSerializedMessage.from_yaml_element(el.id, el.name, FluentAstAttributeFactory.from_yaml_element(el), el.parent_id), yaml_elements) - ) + fluent_serialized_messages = list(map(YAMLExtractor.serialize_yaml_element, yaml_elements)) fluent_exist_serialized_messages = list(filter(lambda m: m, fluent_serialized_messages)) if not len(fluent_exist_serialized_messages): @@ -49,7 +62,6 @@ def create_en_fluent_file(self, relative_parent_dir, file_name, file_data): en_new_dir_path = os.path.join(project.en_locale_prototypes_dir_path, relative_parent_dir) en_fluent_file = FluentFile(os.path.join(en_new_dir_path, f'{file_name}.ftl')) en_fluent_file.save_data(file_data) - logging.info(f'Актуализирован файл английской локали {en_fluent_file.full_path}') return en_fluent_file.full_path diff --git a/Tools/ss14_ru/yamlmodels.py b/Tools/ss14_ru/yamlmodels.py index 0c35e9af719..710ee72a610 100644 --- a/Tools/ss14_ru/yamlmodels.py +++ b/Tools/ss14_ru/yamlmodels.py @@ -1,4 +1,4 @@ -class YAMLEntity: +class YAMLEntity: def __init__(self, id, name, description, suffix, parent_id = None): self.id = id self.name = name From 51e4b1b46fc43cf63ce357c095933f39c2f2d4dd Mon Sep 17 00:00:00 2001 From: Evgencheg <73418250+Evgencheg@users.noreply.github.com> Date: Sun, 6 Oct 2024 16:12:23 +0300 Subject: [PATCH 2/6] Update Tools/ss14_ru/clean_duplicates.py Co-authored-by: mhamster <81412348+mhamsterr@users.noreply.github.com> --- Tools/ss14_ru/clean_duplicates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/ss14_ru/clean_duplicates.py b/Tools/ss14_ru/clean_duplicates.py index daea65b4bc9..b43909e8a3b 100644 --- a/Tools/ss14_ru/clean_duplicates.py +++ b/Tools/ss14_ru/clean_duplicates.py @@ -14,6 +14,7 @@ def find_top_level_dir(start_dir): print(f"Не удалось найти {marker_file} начиная с {start_dir}") exit(-1) current_dir = parent_dir + def find_ftl_files(root_dir): ftl_files = [] for root, dirs, files in os.walk(root_dir): From c99f33bd952de4272206edb957ea3a63e7052636 Mon Sep 17 00:00:00 2001 From: Evgencheg <73418250+Evgencheg@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:15:29 +0300 Subject: [PATCH 3/6] Update translation.sh --- Tools/ss14_ru/translation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/ss14_ru/translation.sh b/Tools/ss14_ru/translation.sh index 22affc3099d..db231c7d4cf 100644 --- a/Tools/ss14_ru/translation.sh +++ b/Tools/ss14_ru/translation.sh @@ -5,7 +5,7 @@ if [ "$(dirname $0)" != "." ]; then cd "$(dirname $0)" fi -pip install -r requirements.txt +pip install -r requirements.txt --no-warn-script-location python3 ./yamlextractor.py python3 ./keyfinder.py python3 ./clean_duplicates.py From b1f5561bce630e298fba7987cbdec4212e1932d9 Mon Sep 17 00:00:00 2001 From: Evgencheg <73418250+Evgencheg@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:15:46 +0300 Subject: [PATCH 4/6] Update translation.bat --- Tools/ss14_ru/translation.bat | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Tools/ss14_ru/translation.bat b/Tools/ss14_ru/translation.bat index 9b614730c6d..b21dfd484fd 100644 --- a/Tools/ss14_ru/translation.bat +++ b/Tools/ss14_ru/translation.bat @@ -1,7 +1,9 @@ @echo off -call pip install -r requirements.txt +call pip install -r requirements.txt --no-warn-script-location call python ./yamlextractor.py call python ./keyfinder.py call python ./clean_duplicates.py call python ./clean_empty.py + +PAUSE From 34bb2f0680792fc3443bdbe3516006240317357b Mon Sep 17 00:00:00 2001 From: Evgencheg <73418250+Evgencheg@users.noreply.github.com> Date: Thu, 10 Oct 2024 22:18:10 +0300 Subject: [PATCH 5/6] Update requirements.txt --- Tools/ss14_ru/requirements.txt | Bin 146 -> 84 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/Tools/ss14_ru/requirements.txt b/Tools/ss14_ru/requirements.txt index bb76cba1b5ec9319e7664e7a15330c80b1b49758..7ee7942b36872a514104157ca0a5019f71b4e8a0 100644 GIT binary patch literal 84 zcmaFAdw*I^X=+}Ht*wEcp@E(OFIPcjN@8(_t*wO~P=uE&pfb|Y*T>e@48kp`EXd4D gk58>ANzE(H%+D*fwKdVR1nEo8NGwW8E#c(?0EU4X`v3p{ literal 146 zcmezWFO4CGp_Cz&A&;Si!4?P&81xtn!K48LF9R1t0YfE23PU18F+&De-U28OQ3H|< z0Ln%(I5PM$_<&{2P-IJhvIPv8K(o^s;(>Nl0PO(TR16f$2h$)MO@MkV!TLZZ0sshE B7M%b9 From cb7e982efa84ff0498ba8d5aa15c0c400b69827a Mon Sep 17 00:00:00 2001 From: Morb0 <14136326+Morb0@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:39:24 +0300 Subject: [PATCH 6/6] Fix CRLF requirements.txt --- Tools/ss14_ru/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tools/ss14_ru/requirements.txt b/Tools/ss14_ru/requirements.txt index 7ee7942b368..dae3410246f 100644 --- a/Tools/ss14_ru/requirements.txt +++ b/Tools/ss14_ru/requirements.txt @@ -1,5 +1,5 @@ -fluent==0.10.0 -pydash==8.0.1 -PyYAML==6.0.1 -typing_extensions==4.9.0 -chardet +fluent==0.10.0 +pydash==8.0.1 +PyYAML==6.0.1 +typing_extensions==4.9.0 +chardet