From 18d074f7d15daa9ad57d1c487b1245a7351977f3 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 27 Oct 2022 14:34:31 +0200 Subject: [PATCH 01/34] Init 1 test later --- .gitignore | 5 +- pytransifex/__init__.py | 3 - pytransifex/__main__.py | 4 + pytransifex/api.py | 345 +++++++++++++++++++++++--------------- pytransifex/api_new.py | 142 ++++++++++++++++ pytransifex/cli.py | 26 +++ pytransifex/config.py | 22 +++ pytransifex/interfaces.py | 63 +++++++ requirements-locked.txt | 85 ++++++++++ requirements.txt | 2 + test/test_translation.py | 43 ----- tests/__init__.py | 0 tests/test_cli.py | 19 +++ tests/test_new_api.py | 23 +++ tests/test_translation.py | 45 +++++ 15 files changed, 649 insertions(+), 178 deletions(-) create mode 100644 pytransifex/__main__.py create mode 100644 pytransifex/api_new.py create mode 100644 pytransifex/cli.py create mode 100644 pytransifex/config.py create mode 100644 pytransifex/interfaces.py create mode 100644 requirements-locked.txt delete mode 100644 test/test_translation.py create mode 100644 tests/__init__.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_new_api.py create mode 100644 tests/test_translation.py diff --git a/.gitignore b/.gitignore index 84ef83e..d5ae537 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ __pycache__/ venv/ .idea -.DS_Store \ No newline at end of file +.DS_Store +.vscode/settings.json +Pipfile +Pipfile.lock diff --git a/pytransifex/__init__.py b/pytransifex/__init__.py index 958add5..139597f 100755 --- a/pytransifex/__init__.py +++ b/pytransifex/__init__.py @@ -1,5 +1,2 @@ - -from pytransifex.api import Transifex -from pytransifex.exceptions import TransifexException, PyTransifexException, InvalidSlugException diff --git a/pytransifex/__main__.py b/pytransifex/__main__.py new file mode 100644 index 0000000..d0b51e4 --- /dev/null +++ b/pytransifex/__main__.py @@ -0,0 +1,4 @@ +from cli import run_cli + +if __name__ == "__main__": + run_cli() diff --git a/pytransifex/api.py b/pytransifex/api.py index 35c9e00..810e290 100755 --- a/pytransifex/api.py +++ b/pytransifex/api.py @@ -1,17 +1,36 @@ -#/usr/bin/python3 - - import os import codecs import requests import json +from typing import Any +from types import FunctionType + +from pytransifex.config import Config from pytransifex.exceptions import PyTransifexException -class Transifex(object): - def __init__(self, api_token: str, organization: str, i18n_type: str = 'PO'): +class Transifex: + transifex = None + + @classmethod + def get(cls, config: None | Config = None) -> "Transifex": + if cls.transifex: + return cls.transifex + if config: + cls.transifex = cls(config) + return cls.transifex + raise PyTransifexException( + "Need to initialize the program with a working configuration" + ) + + @classmethod + @property + def list_funcs(cls) -> list[str]: + return [n for n, f in cls.__dict__.items() if isinstance(f, FunctionType)] + + def __init__(self, config: Config): """ - Initializes Transifex + Initializes Transifex Parameters ---------- @@ -23,26 +42,46 @@ def __init__(self, api_token: str, organization: str, i18n_type: str = 'PO'): the type of translation (PO, QT, …) defaults to: PO """ - self.auth = ('api', api_token) - self.api_key = api_token - self.organization = organization - self.i18n_type = i18n_type - - def create_project(self, - slug, - name: str = None, - source_language_code: str = 'en-gb', - outsource_project_name: str = None, - private: bool = False, - repository_url: str = None): + self.auth = ("api", config.api_token) + self.api_key = config.api_token + self.organization = config.organization + self.i18n_type = config.i18n_type + + def exec(self, fn_name: str, args: dict[str, Any]) -> Any: + error = "" + + if not fn_name in self.list_funcs: + defined = "\n".join(self.list_funcs) + error += f"This function {fn_name} is not defined. Defined are {defined}" + + if "dry_run" in args and args["dry_run"]: + return error or f"Dry run: Would be calling {fn_name} with {args}." + + if error: + raise PyTransifexException(error) + + try: + return getattr(self, fn_name)(**args) + except Exception as error: + return str(error) + + def create_project( + self, + project_slug: str, + project_name: str | None = None, + source_language_code: str = "en-gb", + outsource_project_name: str | None = None, + private: bool = False, + repository_url: str | None = None, + ): """ Create a new project on Transifex - + Parameters ---------- - slug + project_slug the project slug - name + project_name the project name, defaults to the project slug source_language_code the source language code, defaults to 'en-gb' @@ -60,49 +99,50 @@ def create_project(self, `PyTransifexException` if project was not created properly """ - if name is None: - name = slug - - url = 'https://rest.api.transifex.com/projects' - data = { - 'data': { - 'attributes': { - 'name': name, - 'slug': slug, - 'description': name, - 'private': private, - 'repository_url': repository_url + if project_name is None: + project_name = project_slug + + url = "https://rest.api.transifex.com/projects" + data: dict[str, Any] = { + "data": { + "attributes": { + "name": project_name, + "slug": project_slug, + "description": project_name, + "private": private, + "repository_url": repository_url, }, - 'relationships': { - 'organization': { - 'data': { - 'id': 'o:{}'.format(self.organization), - 'type': 'organizations' + "relationships": { + "organization": { + "data": { + "id": "o:{}".format(self.organization), + "type": "organizations", } }, - 'source_language': { - 'data': { + "source_language": { + "data": { "id": "l:{}".format(source_language_code), - "type": "languages" + "type": "languages", } - } + }, }, - 'type': 'projects' + "type": "projects", } - } if outsource_project_name is not None: - data['outsource'] = outsource_project_name + data["outsource"] = outsource_project_name - response = requests.post(url, - data=json.dumps(data), - headers={ - 'Content-Type': 'application/vnd.api+json', - 'Authorization': 'Bearer {}'.format(self.api_key) - }) + response = requests.post( + url, + data=json.dumps(data), + headers={ + "Content-Type": "application/vnd.api+json", + "Authorization": "Bearer {}".format(self.api_key), + }, + ) - if response.status_code != requests.codes['OK']: - print('Could not create project with data: {}'.format(data)) + if response.status_code != requests.codes["OK"]: + print("Could not create project with data: {}".format(data)) raise PyTransifexException(response) def delete_project(self, project_slug: str): @@ -118,9 +158,17 @@ def delete_project(self, project_slug: str): ------ `PyTransifexException` """ - url = 'https://rest.api.transifex.com/projects/o:{o}:p:{p}'.format(o=self.organization, p=project_slug) - response = requests.delete(url, headers={'Content-Type': 'application/vnd.api+json','Authorization': 'Bearer {}'.format(self.api_key)}) - if response.status_code != requests.codes['OK']: + url = "https://rest.api.transifex.com/projects/o:{o}:p:{p}".format( + o=self.organization, p=project_slug + ) + response = requests.delete( + url, + headers={ + "Content-Type": "application/vnd.api+json", + "Authorization": "Bearer {}".format(self.api_key), + }, + ) + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) def list_resources(self, project_slug) -> list: @@ -146,27 +194,33 @@ def list_resources(self, project_slug) -> list: ------ `PyTransifexException` """ - url = 'https://api.transifex.com/organizations/{o}/projects/{p}/resources/'.format( + url = "https://api.transifex.com/organizations/{o}/projects/{p}/resources/".format( o=self.organization, p=project_slug ) response = requests.get(url, auth=self.auth) - if response.status_code != requests.codes['OK']: + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) - return json.loads(codecs.decode(response.content, 'utf-8')) + return json.loads(codecs.decode(response.content, "utf-8")) def delete_team(self, team_slug: str): - url = 'https://rest.api.transifex.com/teams/o:{o}:t:{t}'.format(o=self.organization, t=team_slug) - response = requests.delete(url, headers={'Content-Type': 'application/vnd.api+json','Authorization': 'Bearer {}'.format(self.api_key)}) - if response.status_code != requests.codes['OK']: + url = "https://rest.api.transifex.com/teams/o:{o}:t:{t}".format( + o=self.organization, t=team_slug + ) + response = requests.delete( + url, + headers={ + "Content-Type": "application/vnd.api+json", + "Authorization": "Bearer {}".format(self.api_key), + }, + ) + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) - def create_resource(self, - project_slug, - path_to_file, - resource_slug=None, - resource_name=None): + def create_resource( + self, project_slug, path_to_file, resource_slug=None, resource_name=None + ): """ Creates a new resource with the specified slug from the given file. @@ -186,21 +240,23 @@ def create_resource(self, `PyTransifexException` `IOError` """ - url = 'https://www.transifex.com/api/2/project/{p}/resources/'.format(p=project_slug) + url = "https://www.transifex.com/api/2/project/{p}/resources/".format( + p=project_slug + ) data = { - 'name': resource_name or resource_slug, - 'slug': resource_slug, - 'i18n_type': self.i18n_type, - 'content': open(path_to_file, 'r').read() + "name": resource_name or resource_slug, + "slug": resource_slug, + "i18n_type": self.i18n_type, + "content": open(path_to_file, "r").read(), } response = requests.post( url, data=json.dumps(data), auth=self.auth, - headers={'content-type': 'application/json'} + headers={"content-type": "application/json"}, ) - if response.status_code != requests.codes['CREATED']: + if response.status_code != requests.codes["CREATED"]: raise PyTransifexException(response) def delete_resource(self, project_slug, resource_slug): @@ -218,13 +274,14 @@ def delete_resource(self, project_slug, resource_slug): ------ `PyTransifexException` """ - url = '{u}/project/{s}/resource/{r}'.format(u=self._base_api_url, s=project_slug, r=resource_slug) + url = "{u}/project/{s}/resource/{r}".format( + u=self._base_api_url, s=project_slug, r=resource_slug + ) response = requests.delete(url, auth=self.auth) - if response.status_code != requests.codes['NO_CONTENT']: + if response.status_code != requests.codes["NO_CONTENT"]: raise PyTransifexException(response) - def update_source_translation(self, project_slug, resource_slug, - path_to_file): + def update_source_translation(self, project_slug, resource_slug, path_to_file): """ Update the source translation for a give resource @@ -250,22 +307,26 @@ def update_source_translation(self, project_slug, resource_slug, `PyTransifexException` `IOError` """ - url = 'https://www.transifex.com/api/2/project/{s}/resource/{r}/content'.format( + url = "https://www.transifex.com/api/2/project/{s}/resource/{r}/content".format( s=project_slug, r=resource_slug ) - content = open(path_to_file, 'r').read() - data = {'content': content} + content = open(path_to_file, "r").read() + data = {"content": content} response = requests.put( - url, data=json.dumps(data), auth=self.auth, headers={'content-type': 'application/json'}, + url, + data=json.dumps(data), + auth=self.auth, + headers={"content-type": "application/json"}, ) - if response.status_code != requests.codes['OK']: + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) else: - return json.loads(codecs.decode(response.content, 'utf-8')) + return json.loads(codecs.decode(response.content, "utf-8")) - def create_translation(self, project_slug, resource_slug, language_code, - path_to_file) -> dict: + def create_translation( + self, project_slug, resource_slug, language_code, path_to_file + ) -> dict: """ Creates or updates the translation for the specified language @@ -293,25 +354,30 @@ def create_translation(self, project_slug, resource_slug, language_code, `PyTransifexException` `IOError` """ - url = 'https://www.transifex.com/api/2/project/{s}/resource/{r}/translation/{l}'.format( + url = "https://www.transifex.com/api/2/project/{s}/resource/{r}/translation/{l}".format( s=project_slug, r=resource_slug, l=language_code ) - content = open(path_to_file, 'r').read() - data = {'content': content} + content = open(path_to_file, "r").read() + data = {"content": content} response = requests.put( - url, data=json.dumps(data), auth=self.auth, headers={'content-type': 'application/json'}, + url, + data=json.dumps(data), + auth=self.auth, + headers={"content-type": "application/json"}, ) - if response.status_code != requests.codes['OK']: + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) else: - return json.loads(codecs.decode(response.content, 'utf-8')) - - def get_translation(self, - project_slug: str, - resource_slug: str, - language_code: str, - path_to_file: str): + return json.loads(codecs.decode(response.content, "utf-8")) + + def get_translation( + self, + project_slug: str, + resource_slug: str, + language_code: str, + path_to_file: str, + ): """ Returns the requested translation, if it exists. The translation is returned as a serialized string, unless the GET parameter file is @@ -335,15 +401,16 @@ def get_translation(self, `PyTransifexException` `IOError` """ - url = 'https://www.transifex.com/api/2/project/{s}/resource/{r}/translation/{l}'.format( - s=project_slug, r=resource_slug, l=language_code) - query = {'file': ''} + url = "https://www.transifex.com/api/2/project/{s}/resource/{r}/translation/{l}".format( + s=project_slug, r=resource_slug, l=language_code + ) + query = {"file": ""} response = requests.get(url, auth=self.auth, params=query) - if response.status_code != requests.codes['OK']: + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) else: os.makedirs(os.path.dirname(path_to_file), exist_ok=True) - with open(path_to_file, 'wb') as f: + with open(path_to_file, "wb") as f: for line in response.iter_content(): f.write(line) @@ -367,17 +434,21 @@ def list_languages(self, project_slug, resource_slug): ------ `PyTransifexException` """ - url = 'https://www.transifex.com/api/2/project/{s}/resource/{r}'.format(s=project_slug, r=resource_slug) - response = requests.get(url, auth=self.auth, params={'details': ''}) + url = "https://www.transifex.com/api/2/project/{s}/resource/{r}".format( + s=project_slug, r=resource_slug + ) + response = requests.get(url, auth=self.auth, params={"details": ""}) - if response.status_code != requests.codes['OK']: + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) - content = json.loads(codecs.decode(response.content, 'utf-8')) - languages = [language['code'] for language in content['available_languages']] + content = json.loads(codecs.decode(response.content, "utf-8")) + languages = [language["code"] for language in content["available_languages"]] return languages - def create_language(self, project_slug: str, language_code: str, coordinators: list): + def create_language( + self, project_slug: str, language_code: str, coordinators: list + ): """ Create a new language for the given project Parameters @@ -387,16 +458,20 @@ def create_language(self, project_slug: str, language_code: str, coordinators: l coordinators: list of coordinators """ - url = 'https://www.transifex.com/api/2/project/{s}/languages'.format(s=project_slug) - data = {'language_code': language_code, 'coordinators': coordinators} - response = requests.post(url, - headers={'content-type': 'application/json'}, - auth=self.auth, - data=json.dumps(data)) - if response.status_code != requests.codes['CREATED']: + url = "https://www.transifex.com/api/2/project/{s}/languages".format( + s=project_slug + ) + data = {"language_code": language_code, "coordinators": coordinators} + response = requests.post( + url, + headers={"content-type": "application/json"}, + auth=self.auth, + data=json.dumps(data), + ) + if response.status_code != requests.codes["CREATED"]: raise PyTransifexException(response) - def coordinator(self, project_slug: str, language_code: str = 'en') -> str: + def coordinator(self, project_slug: str, language_code: str = "en") -> str: """ Return the coordinator of the the project @@ -405,12 +480,14 @@ def coordinator(self, project_slug: str, language_code: str = 'en') -> str: project_slug: language_code: """ - url = 'https://www.transifex.com/api/2/project/{s}/language/{l}/coordinators'.format(s=project_slug, l=language_code) + url = "https://www.transifex.com/api/2/project/{s}/language/{l}/coordinators".format( + s=project_slug, l=language_code + ) response = requests.get(url, auth=self.auth) - if response.status_code != requests.codes['OK']: + if response.status_code != requests.codes["OK"]: raise PyTransifexException(response) - content = json.loads(codecs.decode(response.content, 'utf-8')) - return content['coordinators'] + content = json.loads(codecs.decode(response.content, "utf-8")) + return content["coordinators"] def project_exists(self, project_slug) -> bool: """ @@ -422,15 +499,19 @@ def project_exists(self, project_slug) -> bool: project_slug The project slug """ - url = 'https://rest.api.transifex.com/projects/o:{o}:p:{s}'.format(o=self.organization, s=project_slug) - response = requests.get(url, - headers={ - 'Content-Type': 'application/vnd.api+json', - 'Authorization': 'Bearer {}'.format(self.api_key) - }) - if response.status_code == requests.codes['OK']: + url = "https://rest.api.transifex.com/projects/o:{o}:p:{s}".format( + o=self.organization, s=project_slug + ) + response = requests.get( + url, + headers={ + "Content-Type": "application/vnd.api+json", + "Authorization": "Bearer {}".format(self.api_key), + }, + ) + if response.status_code == requests.codes["OK"]: return True - elif response.status_code == requests.codes['NOT_FOUND']: + elif response.status_code == requests.codes["NOT_FOUND"]: return False else: raise PyTransifexException(response) @@ -439,9 +520,11 @@ def ping(self) -> bool: """ Check the connection to the server and the auth credentials """ - url = 'https://api.transifex.com/organizations/{}/projects/'.format(self.organization) + url = "https://api.transifex.com/organizations/{}/projects/".format( + self.organization + ) response = requests.get(url, auth=self.auth) - success = response.status_code == requests.codes['OK'] + success = response.status_code == requests.codes["OK"] if not success: raise PyTransifexException(response) return success diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py new file mode 100644 index 0000000..255552e --- /dev/null +++ b/pytransifex/api_new.py @@ -0,0 +1,142 @@ +from types import FunctionType +from typing import Any +from transifex.api import transifex_api as tx_api + +from pytransifex.config import Config +from pytransifex.exceptions import PyTransifexException + +base_url: str = "https://rest.api.transifex.com" + + +class TransifexNew: + # TODO + # This class satisfies the interface expected by qgis-plugin-cli + # but it falls short of implementing all the methods defined in + # pytransifex.api_old.py. The question is whether I should implement them. + # Is there's a consumer downstream? + + @classmethod + def get(cls, config: None | Config = None) -> "TransifexNew": + if cls.transifex: + return cls.transifex + if config: + cls.transifex = cls(config) + return cls.transifex + raise PyTransifexException( + "Need to initialize the program with a working configuration" + ) + + @classmethod + @property + def list_funcs(cls) -> list[str]: + return [n for n, f in cls.__dict__.items() if isinstance(f, FunctionType)] + + def __init__(self, config: Config): + """Extract config values, consumes API token against SDK client""" + self.organization = config.organization + self.i18n_type = config.i18n_type + self.client = tx_api + self.client.setup(auth=config.api_token) + + """ Set up some state to minimize network round-trips """ + self._organization_api_object = self.client.Organization.get(slug=config.organization) + + def exec(self, fn_name: str, args: dict[str, Any]) -> Any: + """Adapter for this class to be used from the CLI module""" + error = "" + + if not fn_name in self.list_funcs: + defined = "\n".join(self.list_funcs) + error += f"This function {fn_name} is not defined. Defined are {defined}" + + if "dry_run" in args and args["dry_run"]: + return error or f"Dry run: Would be calling {fn_name} with {args}." + + if error: + raise PyTransifexException(error) + + try: + return getattr(self, fn_name)(**args) + except Exception as error: + return str(error) + + def create_project( + self, + project_slug: str, + project_name: str | None = None, + source_language_code: str = "en-gb", + # FIXME: Not sure it's possible to use this param with the new API + outsource_project_name: str | None = None, + private: bool = False, + repository_url: str | None = None, + ): + _ = self.client.Project.create( + name=project_name, + slug=project_slug, + private=private, + organization=self.organization, + source_language=source_language_code, + repository_url=repository_url, + ) + + def list_resources(self, project_slug: str) -> list[Any]: + if projects := self._organization_api_object.fetch("projects"): + return projects.filter(slug=project_slug) + return [] + + def create_resource( + self, + # FIXME + # Unused + project_slug: str, + path_to_file: str, + resource_slug: str | None = None, + resource_name: str | None = None, + ): + # FIXME How to name a to-be-created resource if both resource_slug and resource_name are None? + slug = resource_slug or resource_name + resource = self.client.Resource.get(slug=slug) + + if not slug: + raise PyTransifexException( + "Please give either a resource_slug or resource_name" + ) + + if not resource: + raise PyTransifexException( + f"Unable to find any resource associated with {slug}" + ) + + with open(path_to_file, "r") as handler: + content = handler.read() + # self.client.Resource.create(...) + self.client.ResourceStringsAsyncUpload.upload(resource, content) + + def update_source_translation( + self, project_slug: str, resource_slug: str, path_to_file: str + ): + resource = self.client.Resource.get(slug=slug) + + with open(path_to_file, "r") as handler: + content = handler.read() + self.client.ResourceTranslationsAsyncUpload(resource, content) + + def get_translation( + self, project_slug: str, resource_slug: str, language: str, path_to_file: str + ): + ... + + def list_languages(self, project_slug: str, resource_slug: str) -> list[Any]: + ... + + def create_language(self, project_slug: str, path_to_file: str, resource_slug: str): + ... + + def project_exists(self, project_slug: str) -> bool: + if organization := self.client.Organization.get(slug=project_slug): + if organization.fetch("projects"): + return True + return False + + def ping(self): + ... diff --git a/pytransifex/cli.py b/pytransifex/cli.py new file mode 100644 index 0000000..9fb1552 --- /dev/null +++ b/pytransifex/cli.py @@ -0,0 +1,26 @@ +from typing import Any +import click + +from pytransifex.api import Transifex + + +def format_args(args: dict[str, Any]) -> dict[str, Any]: + return {k.replace("-", "_"): v for k, v in args.items()} + + +@click.command() +@click.argument("f_name") +@click.option("--dry-run", type=bool) +@click.option("--outsource-project-name", type=str) +@click.option("--private", type=bool) +@click.option("--path-to-file", type=str) +@click.option("--project-name", type=str) +@click.option("--project-slug", type=str) +@click.option("--repository-url", type=str) +@click.option("--resource-name", type=str) +@click.option("--resource-slug", type=str) +@click.option("--source-lang-code", type=str) +def run_cli(f_name: str, **options): + translator = Transifex.get() + result = translator.exec(f_name, format_args(options)) + click.echo(result) diff --git a/pytransifex/config.py b/pytransifex/config.py new file mode 100644 index 0000000..5bc53e4 --- /dev/null +++ b/pytransifex/config.py @@ -0,0 +1,22 @@ +from os import environ +from typing import NamedTuple +from pytransifex.exceptions import PyTransifexException + + +class Config(NamedTuple): + api_token: str + organization: str + i18n_type: str + + @classmethod + def from_env(cls) -> "Config": + token = environ.get("TX_API_TOKEN") + organization = environ.get("ORGANIZATION") + i18n_type = environ.get("I18N_TYPE", "PO") + + if any(v is None for v in [token, organization]): + raise PyTransifexException( + "Both 'TX_API_TOKEN' and 'ORGANIZATION' must be set environment variables. Aborting now." + ) + + return cls(token, organization, i18n_type) # type: ignore diff --git a/pytransifex/interfaces.py b/pytransifex/interfaces.py new file mode 100644 index 0000000..e1a1353 --- /dev/null +++ b/pytransifex/interfaces.py @@ -0,0 +1,63 @@ +from typing import Any, Protocol, runtime_checkable + +from pytransifex.config import Config + + +@runtime_checkable +class IsTranslator(Protocol): + @classmethod + def get(cls, config: None | Config = None): + ... + + @classmethod + @property + def list_funcs(cls) -> list[str]: + ... + + def exec(self, fn_name: str, args: dict[str, Any]) -> Any: + ... + + def ping(self): + ... + + def project_exists(self, project: str) -> bool: + ... + + def create_project( + self, + project_slug: str, + project_name: str | None = None, + source_language_code: str = "en-gb", + outsource_project_name: str | None = None, + private: bool = False, + repository_url: str | None = None, + ): + ... + + def list_resources(self, project_slug: str) -> list[Any]: + ... + + def create_resource( + self, + project_slug: str, + path_to_file: str, + resource_slug: str | None = None, + resource_name: str | None = None, + ): + ... + + def update_source_translation( + self, project_slug: str, resource_slug: str, path_to_file: str + ): + ... + + def get_translation( + self, project_slug: str, resource_slug: str, language: str, path_to_file: str + ): + ... + + def list_languages(self, project_slug: str, resource_slug: str) -> list[Any]: + ... + + def create_language(self, project_slug: str, path_to_file: str, resource_slug: str): + ... diff --git a/requirements-locked.txt b/requirements-locked.txt new file mode 100644 index 0000000..961b398 --- /dev/null +++ b/requirements-locked.txt @@ -0,0 +1,85 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --output-file=requirements-locked.txt requirements.txt +# +alabaster==0.7.12 + # via sphinx +asttokens==2.0.8 + # via transifex-python +babel==2.10.3 + # via sphinx +certifi==2022.9.24 + # via requests +charset-normalizer==2.1.1 + # via requests +click==8.1.3 + # via + # -r requirements.txt + # transifex-python +docutils==0.17.1 + # via + # sphinx + # sphinx-rtd-theme +future==0.18.2 + # via pyseeyou +idna==3.4 + # via requests +imagesize==1.4.1 + # via sphinx +jinja2==3.1.2 + # via sphinx +markupsafe==2.1.1 + # via jinja2 +nose2==0.12.0 + # via -r requirements.txt +packaging==21.3 + # via sphinx +parsimonious==0.10.0 + # via pyseeyou +pygments==2.13.0 + # via sphinx +pyparsing==3.0.9 + # via packaging +pyseeyou==1.0.2 + # via transifex-python +pytz==2022.5 + # via + # babel + # transifex-python +regex==2022.9.13 + # via parsimonious +requests==2.28.1 + # via + # -r requirements.txt + # sphinx + # transifex-python +six==1.16.0 + # via asttokens +snowballstemmer==2.2.0 + # via sphinx +sphinx==5.3.0 + # via + # -r requirements.txt + # sphinx-rtd-theme +sphinx-rtd-theme==1.0.0 + # via -r requirements.txt +sphinxcontrib-applehelp==1.0.2 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.0 + # via sphinx +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +toolz==0.12.0 + # via pyseeyou +transifex-python==3.0.2 + # via -r requirements.txt +urllib3==1.26.12 + # via requests diff --git a/requirements.txt b/requirements.txt index c6e8990..8601b34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ requests sphinx sphinx_rtd_theme nose2 +transifex-python +click \ No newline at end of file diff --git a/test/test_translation.py b/test/test_translation.py deleted file mode 100644 index 3b334ff..0000000 --- a/test/test_translation.py +++ /dev/null @@ -1,43 +0,0 @@ -#! /usr/bin/env python - -import unittest -import os - -from pytransifex import Transifex, PyTransifexException - - -class TestTranslation(unittest.TestCase): - - def setUp(self): - token = os.getenv('TX_TOKEN') - assert token is not None - self.t = Transifex(organization='pytransifex', api_token=token) - self.project_slug = 'pytransifex-test-project' - self.project_name = 'PyTransifex Test project' - self.source_lang = 'fr_FR' - - def tearDown(self): - try: - self.t.delete_project(self.project_slug) - except PyTransifexException: - pass - try: - self.t.delete_team('{}-team'.format(self.project_slug)) - except PyTransifexException: - pass - - def test_creation(self): - self.tearDown() - self.t.create_project( - name=self.project_name, - slug=self.project_slug, - source_language_code=self.source_lang, - private=False, - repository_url='https://www.github.com/opengisch/pytransifex' - ) - - self.assertTrue(self.t.project_exists(self.project_slug)) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..a8bc54e --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,19 @@ +import unittest +from click.testing import CliRunner + +from pytransifex.config import Config +from pytransifex.cli import run_cli +from pytransifex.api import Transifex + + +class TestCli(unittest.TestCase): + def test_cli(self): + config = Config("token", "organization", "po") + _ = Transifex.get(config) + runner = CliRunner() + result = runner.invoke(run_cli, ["create_project", "--dry-run", "true"]) + assert result.exit_code == 0 + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_new_api.py b/tests/test_new_api.py new file mode 100644 index 0000000..dd06351 --- /dev/null +++ b/tests/test_new_api.py @@ -0,0 +1,23 @@ +import unittest + +from pytransifex.config import Config +from pytransifex.api import Transifex +from pytransifex.api_new import TransifexNew +from pytransifex.interfaces import IsTranslator + + +config = Config("tok", "orga", "ln") + + +class TestNewApi(unittest.TestCase): + def test_old_api_satisfies_protocol(self): + old_api = Transifex(config) + assert isinstance(old_api, IsTranslator) + + def test_new_api_satisfies_protocol(self): + new_api = TransifexNew(config) + assert isinstance(new_api, IsTranslator) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_translation.py b/tests/test_translation.py new file mode 100644 index 0000000..b1e8e04 --- /dev/null +++ b/tests/test_translation.py @@ -0,0 +1,45 @@ +#! /usr/bin/env python + +import unittest +import os + +from pytransifex import Config, PyTransifexException +from pytransifex.api import Transifex + + +class TestTranslation(unittest.TestCase): + def setUp(self): + token = os.getenv("TX_TOKEN") + assert token is not None + self.t = Transifex( + Config(organization="pytransifex", api_token=token, i18n_type="PO") + ) + self.project_slug = "pytransifex-test-project" + self.project_name = "PyTransifex Test project" + self.source_lang = "fr_FR" + + def tearDown(self): + try: + self.t.delete_project(self.project_slug) + except PyTransifexException: + pass + try: + self.t.delete_team("{}-team".format(self.project_slug)) + except PyTransifexException: + pass + + def test_creation(self): + self.tearDown() + self.t.create_project( + project_name=self.project_name, + project_slug=self.project_slug, + source_language_code=self.source_lang, + private=False, + repository_url="https://www.github.com/opengisch/pytransifex", + ) + + self.assertTrue(self.t.project_exists(self.project_slug)) + + +if __name__ == "__main__": + unittest.main() From ddd66d5ad8de3d78bfe27604935c9eed3195f4d9 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Fri, 28 Oct 2022 13:51:51 +0200 Subject: [PATCH 02/34] Protocol -> ABC for better checks in method signatures. --- pytransifex/api.py | 5 +- pytransifex/api_new.py | 65 ++++++++++++------- pytransifex/cli.py | 43 +++++++----- pytransifex/interfaces.py | 65 ++++++++++++++----- pytransifex/utils.py | 11 ++++ tests/{test_new_api.py => test_api_new.py} | 8 +-- .../{test_translation.py => test_api_old.py} | 3 +- tests/test_cli.py | 9 ++- 8 files changed, 141 insertions(+), 68 deletions(-) create mode 100644 pytransifex/utils.py rename tests/{test_new_api.py => test_api_new.py} (70%) rename tests/{test_translation.py => test_api_old.py} (92%) diff --git a/pytransifex/api.py b/pytransifex/api.py index 810e290..4a478ca 100755 --- a/pytransifex/api.py +++ b/pytransifex/api.py @@ -7,13 +7,14 @@ from pytransifex.config import Config from pytransifex.exceptions import PyTransifexException +from pytransifex.interfaces import IsTranslator -class Transifex: +class Transifex(IsTranslator): transifex = None @classmethod - def get(cls, config: None | Config = None) -> "Transifex": + def __call__(cls, config: None | Config = None) -> "Transifex": if cls.transifex: return cls.transifex if config: diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 255552e..fd27fbc 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -4,43 +4,37 @@ from pytransifex.config import Config from pytransifex.exceptions import PyTransifexException +from pytransifex.interfaces import IsTranslator +from pytransifex.utils import auth_client base_url: str = "https://rest.api.transifex.com" -class TransifexNew: - # TODO - # This class satisfies the interface expected by qgis-plugin-cli - # but it falls short of implementing all the methods defined in - # pytransifex.api_old.py. The question is whether I should implement them. - # Is there's a consumer downstream? - - @classmethod - def get(cls, config: None | Config = None) -> "TransifexNew": - if cls.transifex: - return cls.transifex - if config: - cls.transifex = cls(config) - return cls.transifex - raise PyTransifexException( - "Need to initialize the program with a working configuration" - ) - +class Client(IsTranslator): @classmethod @property def list_funcs(cls) -> list[str]: return [n for n, f in cls.__dict__.items() if isinstance(f, FunctionType)] - def __init__(self, config: Config): + def __init__(self, config: Config, defer_login: bool = False): """Extract config values, consumes API token against SDK client""" + self.api_token = config.api_token self.organization = config.organization self.i18n_type = config.i18n_type self.client = tx_api - self.client.setup(auth=config.api_token) - - """ Set up some state to minimize network round-trips """ - self._organization_api_object = self.client.Organization.get(slug=config.organization) + self.logged_in = False + + if not defer_login: + self.login() + + def login(self): + self.client.setup(auth=self.api_token) + self._organization_api_object = self.client.Organization.get( + slug=self.organization + ) + self.logged_in = True + @auth_client def exec(self, fn_name: str, args: dict[str, Any]) -> Any: """Adapter for this class to be used from the CLI module""" error = "" @@ -60,6 +54,7 @@ def exec(self, fn_name: str, args: dict[str, Any]) -> Any: except Exception as error: return str(error) + @auth_client def create_project( self, project_slug: str, @@ -79,11 +74,13 @@ def create_project( repository_url=repository_url, ) + @auth_client def list_resources(self, project_slug: str) -> list[Any]: if projects := self._organization_api_object.fetch("projects"): return projects.filter(slug=project_slug) return [] + @auth_client def create_resource( self, # FIXME @@ -112,31 +109,51 @@ def create_resource( # self.client.Resource.create(...) self.client.ResourceStringsAsyncUpload.upload(resource, content) + @auth_client def update_source_translation( self, project_slug: str, resource_slug: str, path_to_file: str ): - resource = self.client.Resource.get(slug=slug) + resource = self.client.Resource.get(slug=resource_slug) with open(path_to_file, "r") as handler: content = handler.read() self.client.ResourceTranslationsAsyncUpload(resource, content) + @auth_client def get_translation( self, project_slug: str, resource_slug: str, language: str, path_to_file: str ): ... + @auth_client def list_languages(self, project_slug: str, resource_slug: str) -> list[Any]: ... + @auth_client def create_language(self, project_slug: str, path_to_file: str, resource_slug: str): ... + @auth_client def project_exists(self, project_slug: str) -> bool: if organization := self.client.Organization.get(slug=project_slug): if organization.fetch("projects"): return True return False + @auth_client def ping(self): ... + + +class Transifex: + client = None + + def __new__(cls, config: Config | None = None, defer_login: bool = False): + if not cls.client: + + if not config: + raise Exception("Need to pass config") + + cls.client = Client(config,defer_login) + + return cls.client diff --git a/pytransifex/cli.py b/pytransifex/cli.py index 9fb1552..3b8693a 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -2,25 +2,36 @@ import click from pytransifex.api import Transifex +from pytransifex.config import Config def format_args(args: dict[str, Any]) -> dict[str, Any]: return {k.replace("-", "_"): v for k, v in args.items()} -@click.command() -@click.argument("f_name") -@click.option("--dry-run", type=bool) -@click.option("--outsource-project-name", type=str) -@click.option("--private", type=bool) -@click.option("--path-to-file", type=str) -@click.option("--project-name", type=str) -@click.option("--project-slug", type=str) -@click.option("--repository-url", type=str) -@click.option("--resource-name", type=str) -@click.option("--resource-slug", type=str) -@click.option("--source-lang-code", type=str) -def run_cli(f_name: str, **options): - translator = Transifex.get() - result = translator.exec(f_name, format_args(options)) - click.echo(result) +import click + + +@click.group() +def cli(): + pass + + +@click.option("--verbose", is_flag=True, default=False) +@click.argument("file_name", required=True) +@cli.command("push", help="Push documentation") +def push(file_name: str, verbose: bool): + click.echo(f"push: {file_name}") + if verbose: + click.echo("Was it verbose enough?") + + +@click.option("--only-lang", "-l", default="all") +@click.argument("dir_name", required=True) +@cli.command("pull", help="Pull documentation") +def pull(dir_name: str, only_lang: str): + click.echo(f"pull: {dir_name}, {only_lang}") + + +if __name__ == "__main__": + cli() diff --git a/pytransifex/interfaces.py b/pytransifex/interfaces.py index e1a1353..a7033e6 100644 --- a/pytransifex/interfaces.py +++ b/pytransifex/interfaces.py @@ -1,28 +1,26 @@ -from typing import Any, Protocol, runtime_checkable +from typing import Any +from abc import abstractclassmethod, abstractmethod, ABC from pytransifex.config import Config -@runtime_checkable -class IsTranslator(Protocol): - @classmethod - def get(cls, config: None | Config = None): - ... +class IsTranslator(ABC): + # TODO + # This interface satisfies the expectations of qgis-plugin-cli + # but it falls short of requiring an implementation for methods + # that qgis-plugin-cli does not use: + # coordinator, create_translation, delete_project, delete_resource, delete_team + # The question is whether I should implement them. Is there's a consumer downstream? - @classmethod - @property + @abstractclassmethod def list_funcs(cls) -> list[str]: - ... + raise NotImplementedError + @abstractmethod def exec(self, fn_name: str, args: dict[str, Any]) -> Any: - ... - - def ping(self): - ... - - def project_exists(self, project: str) -> bool: - ... + raise NotImplementedError + @abstractmethod def create_project( self, project_slug: str, @@ -32,11 +30,19 @@ def create_project( private: bool = False, repository_url: str | None = None, ): + raise NotImplementedError + + def delete_project(self, project_slug: str): ... + @abstractmethod def list_resources(self, project_slug: str) -> list[Any]: + raise NotImplementedError + + def delete_team(self, team_slug: str): ... + @abstractmethod def create_resource( self, project_slug: str, @@ -44,20 +50,43 @@ def create_resource( resource_slug: str | None = None, resource_name: str | None = None, ): + raise NotImplementedError + + def delete_resource(self, project_slug: str, resource_slug: str): ... + @abstractmethod def update_source_translation( self, project_slug: str, resource_slug: str, path_to_file: str ): + raise NotImplementedError + + def create_translation( + self, project_slug: str, language_code: str, path_to_file: str + ) -> dict[str, Any]: ... + @abstractmethod def get_translation( self, project_slug: str, resource_slug: str, language: str, path_to_file: str ): - ... + raise NotImplementedError + @abstractmethod def list_languages(self, project_slug: str, resource_slug: str) -> list[Any]: - ... + raise NotImplementedError + @abstractmethod def create_language(self, project_slug: str, path_to_file: str, resource_slug: str): + raise NotImplementedError + + def coordinator(self, project_slug: str, language_code="str") -> str: ... + + @abstractmethod + def project_exists(self, project_slug: str) -> bool: + raise NotImplementedError + + @abstractmethod + def ping(self): + raise NotImplementedError diff --git a/pytransifex/utils.py b/pytransifex/utils.py new file mode 100644 index 0000000..31fa0df --- /dev/null +++ b/pytransifex/utils.py @@ -0,0 +1,11 @@ +from functools import wraps + + +def auth_client(f): + @wraps(f) + def capture_args(instance, *args, **kwargs): + if not instance.logged_in: + instance.login() + return f(instance, *args, **kwargs) + + return capture_args diff --git a/tests/test_new_api.py b/tests/test_api_new.py similarity index 70% rename from tests/test_new_api.py rename to tests/test_api_new.py index dd06351..96fbaec 100644 --- a/tests/test_new_api.py +++ b/tests/test_api_new.py @@ -1,8 +1,8 @@ import unittest from pytransifex.config import Config -from pytransifex.api import Transifex -from pytransifex.api_new import TransifexNew +from pytransifex.api import Transifex as TOld +from pytransifex.api_new import Transifex as TNew from pytransifex.interfaces import IsTranslator @@ -11,11 +11,11 @@ class TestNewApi(unittest.TestCase): def test_old_api_satisfies_protocol(self): - old_api = Transifex(config) + old_api = TOld(config) assert isinstance(old_api, IsTranslator) def test_new_api_satisfies_protocol(self): - new_api = TransifexNew(config) + new_api = TNew(config, defer_login=True) assert isinstance(new_api, IsTranslator) diff --git a/tests/test_translation.py b/tests/test_api_old.py similarity index 92% rename from tests/test_translation.py rename to tests/test_api_old.py index b1e8e04..87fa3f2 100644 --- a/tests/test_translation.py +++ b/tests/test_api_old.py @@ -3,7 +3,8 @@ import unittest import os -from pytransifex import Config, PyTransifexException +from pytransifex.config import Config +from pytransifex.exceptions import PyTransifexException from pytransifex.api import Transifex diff --git a/tests/test_cli.py b/tests/test_cli.py index a8bc54e..9ca4db2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,16 +2,19 @@ from click.testing import CliRunner from pytransifex.config import Config -from pytransifex.cli import run_cli +from pytransifex.cli import cli from pytransifex.api import Transifex class TestCli(unittest.TestCase): def test_cli(self): config = Config("token", "organization", "po") - _ = Transifex.get(config) + _ = Transifex(config) runner = CliRunner() - result = runner.invoke(run_cli, ["create_project", "--dry-run", "true"]) + result = runner.invoke(cli, ["pull", "somedir", "-l", "fr"]) + + if result.exit_code != 0: + print(result.output) assert result.exit_code == 0 From a31de61f507bbeb7a2c426fe9248b1a08e18df64 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Sun, 30 Oct 2022 18:44:09 +0100 Subject: [PATCH 03/34] CLI build configuration and documentation. --- .gitignore | 1 + README.md | 30 ++++++ pytransifex/__main__.py | 4 - pytransifex/api_new.py | 165 ++++++++++++++--------------- pytransifex/{api.py => api_old.py} | 39 +------ pytransifex/cli.py | 9 +- pytransifex/config.py | 10 +- pytransifex/interfaces.py | 17 +-- pytransifex/utils.py | 2 +- requirements-locked.txt | 2 + requirements.txt | 3 +- setup.py | 52 ++++----- tests/test_api_new.py | 83 +++++++++++++-- tests/test_api_old.py | 2 +- tests/test_cli.py | 2 +- tests/test_resource_fr.po | 1 + 16 files changed, 233 insertions(+), 189 deletions(-) delete mode 100644 pytransifex/__main__.py rename pytransifex/{api.py => api_old.py} (92%) create mode 100644 tests/test_resource_fr.po diff --git a/.gitignore b/.gitignore index d5ae537..08cc41f 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ venv/ .vscode/settings.json Pipfile Pipfile.lock +.env diff --git a/README.md b/README.md index ab7f30d..1fd2f7e 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,33 @@ Read the documentation: https://opengisch.github.io/pytransifex This was largely based on https://github.com/jakul/python-transifex which was not made available for Python 3. + +This can be imported as a library or run as a CLI executable (`pytx`). Read on for the latter use case. + +### Build the package (required for consuming the package as a CLI) + +Ensure that `build` is installed for the current user or project: +- current user: `pip3 --user install build` +- local project: `pipenv install --dev build` + +#### 1. Build the installable archive + +Build the package: +- current user: `pip3 -m build` +- local project: `pipenv run build` + +#### 2. Install from the archive + +This will create a `dist` in the current directory, with a `tar.gz` archive in it. You can finally install it: +- current user: `pip3 --user install ` +- local project: `pipenv install ` + +#### 3. Run the pytx cli + +Once installed the package can be run as a CLI; example: + + pytx pull name -l fr + +Run `pytx --help` for more information. + + diff --git a/pytransifex/__main__.py b/pytransifex/__main__.py deleted file mode 100644 index d0b51e4..0000000 --- a/pytransifex/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from cli import run_cli - -if __name__ == "__main__": - run_cli() diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index fd27fbc..038970a 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,151 +1,146 @@ -from types import FunctionType from typing import Any from transifex.api import transifex_api as tx_api +from transifex.api.jsonapi.resources import Resource from pytransifex.config import Config -from pytransifex.exceptions import PyTransifexException -from pytransifex.interfaces import IsTranslator -from pytransifex.utils import auth_client +from pytransifex.interfaces import Tx +from pytransifex.utils import ensure_logged_client -base_url: str = "https://rest.api.transifex.com" - -class Client(IsTranslator): - @classmethod - @property - def list_funcs(cls) -> list[str]: - return [n for n, f in cls.__dict__.items() if isinstance(f, FunctionType)] +class Client(Tx): + """ + The proper Transifex client expecte by the cli and other consumers. + By default instances are created and logged in 'lazyly' -- when creation and login cannot be deferred any longer. + Methods defined as "..." are left for further discussion; they are not used anywhere in qgis-plugin-cli. + """ def __init__(self, config: Config, defer_login: bool = False): """Extract config values, consumes API token against SDK client""" self.api_token = config.api_token + self.host = config.host self.organization = config.organization self.i18n_type = config.i18n_type - self.client = tx_api self.logged_in = False + self.api = tx_api if not defer_login: self.login() def login(self): - self.client.setup(auth=self.api_token) - self._organization_api_object = self.client.Organization.get( + if self.logged_in: + return + + self.api.setup(host=self.host, auth=self.api_token) + self._organization_api_object = self.api.Organization.get( slug=self.organization ) self.logged_in = True - @auth_client - def exec(self, fn_name: str, args: dict[str, Any]) -> Any: - """Adapter for this class to be used from the CLI module""" - error = "" - - if not fn_name in self.list_funcs: - defined = "\n".join(self.list_funcs) - error += f"This function {fn_name} is not defined. Defined are {defined}" - - if "dry_run" in args and args["dry_run"]: - return error or f"Dry run: Would be calling {fn_name} with {args}." - - if error: - raise PyTransifexException(error) - - try: - return getattr(self, fn_name)(**args) - except Exception as error: - return str(error) - - @auth_client + @ensure_logged_client def create_project( self, project_slug: str, project_name: str | None = None, - source_language_code: str = "en-gb", - # FIXME: Not sure it's possible to use this param with the new API - outsource_project_name: str | None = None, + source_language_code: str = "en_GB", private: bool = False, - repository_url: str | None = None, - ): - _ = self.client.Project.create( + *args, + **kwargs, + ) -> Resource: + """Create a project. args, kwargs are there to absorb unnecessary arguments from consumers.""" + source_language = self.api.Language.get(code=source_language_code) + organization = self._organization_api_object + + return self.api.Project.create( name=project_name, slug=project_slug, + source_language=source_language, private=private, - organization=self.organization, - source_language=source_language_code, - repository_url=repository_url, + organization=organization, ) - @auth_client + @ensure_logged_client + def get_project(self, project_slug: str) -> Resource: + if projects := self._organization_api_object.fetch("projects"): + return projects.get(slug=project_slug) + raise Exception(f"Project not found: {project_slug}") + + @ensure_logged_client def list_resources(self, project_slug: str) -> list[Any]: if projects := self._organization_api_object.fetch("projects"): return projects.filter(slug=project_slug) - return [] + raise Exception(f"Project not found {project_slug}") - @auth_client + @ensure_logged_client def create_resource( self, - # FIXME - # Unused project_slug: str, path_to_file: str, resource_slug: str | None = None, resource_name: str | None = None, ): - # FIXME How to name a to-be-created resource if both resource_slug and resource_name are None? - slug = resource_slug or resource_name - resource = self.client.Resource.get(slug=slug) - - if not slug: - raise PyTransifexException( - "Please give either a resource_slug or resource_name" - ) - - if not resource: - raise PyTransifexException( - f"Unable to find any resource associated with {slug}" - ) - - with open(path_to_file, "r") as handler: - content = handler.read() - # self.client.Resource.create(...) - self.client.ResourceStringsAsyncUpload.upload(resource, content) - - @auth_client + if not (resource_slug or resource_name): + raise Exception("Please give either a resource_slug or resource_name") + + resource = self.api.Resource.create(name=resource_name, slug=resource_slug) + resource.save(project=project_slug) + + file_handler = open(path_to_file, "r") + content = file_handler.read() + file_handler.close() + + self.api.ResourceStringsAsyncUpload.upload(resource, content) + + @ensure_logged_client def update_source_translation( self, project_slug: str, resource_slug: str, path_to_file: str ): - resource = self.client.Resource.get(slug=resource_slug) + resource = self.api.Resource.get(slug=resource_slug, project_slug=project_slug) + file_handler = open(path_to_file, "r") + content = file_handler.read() + file_handler.close() + self.api.ResourceStringsAsyncUpload(resource, content) - with open(path_to_file, "r") as handler: - content = handler.read() - self.client.ResourceTranslationsAsyncUpload(resource, content) - - @auth_client + @ensure_logged_client def get_translation( self, project_slug: str, resource_slug: str, language: str, path_to_file: str ): ... - @auth_client - def list_languages(self, project_slug: str, resource_slug: str) -> list[Any]: - ... + @ensure_logged_client + def list_languages(self, project_slug: str) -> list[Any]: + if projects := self._organization_api_object.fetch("projects"): + if project := projects.get(slug=project_slug): + return project.fetch("languages") + raise Exception(f"Unable to find any data for this project {project_slug}") + raise Exception( + f"You need at least 1 project to be able to retrieve a list of projects." + ) - @auth_client + @ensure_logged_client def create_language(self, project_slug: str, path_to_file: str, resource_slug: str): ... - @auth_client + @ensure_logged_client def project_exists(self, project_slug: str) -> bool: - if organization := self.client.Organization.get(slug=project_slug): - if organization.fetch("projects"): + if projects := self._organization_api_object.fetch("projects"): + if projects.get(slug=project_slug): return True - return False + return False + raise Exception( + f"No project could be found under this organization: {self.organization}" + ) - @auth_client + @ensure_logged_client def ping(self): ... class Transifex: + """ + Singleton factory to ensure the client is initialized at most once. + Simpler to manage than a solution relying on 'imports being imported once in Python. + """ + client = None def __new__(cls, config: Config | None = None, defer_login: bool = False): @@ -154,6 +149,6 @@ def __new__(cls, config: Config | None = None, defer_login: bool = False): if not config: raise Exception("Need to pass config") - cls.client = Client(config,defer_login) + cls.client = Client(config, defer_login) return cls.client diff --git a/pytransifex/api.py b/pytransifex/api_old.py similarity index 92% rename from pytransifex/api.py rename to pytransifex/api_old.py index 4a478ca..b31e62b 100755 --- a/pytransifex/api.py +++ b/pytransifex/api_old.py @@ -7,28 +7,9 @@ from pytransifex.config import Config from pytransifex.exceptions import PyTransifexException -from pytransifex.interfaces import IsTranslator -class Transifex(IsTranslator): - transifex = None - - @classmethod - def __call__(cls, config: None | Config = None) -> "Transifex": - if cls.transifex: - return cls.transifex - if config: - cls.transifex = cls(config) - return cls.transifex - raise PyTransifexException( - "Need to initialize the program with a working configuration" - ) - - @classmethod - @property - def list_funcs(cls) -> list[str]: - return [n for n, f in cls.__dict__.items() if isinstance(f, FunctionType)] - +class Transifex: def __init__(self, config: Config): """ Initializes Transifex @@ -48,24 +29,6 @@ def __init__(self, config: Config): self.organization = config.organization self.i18n_type = config.i18n_type - def exec(self, fn_name: str, args: dict[str, Any]) -> Any: - error = "" - - if not fn_name in self.list_funcs: - defined = "\n".join(self.list_funcs) - error += f"This function {fn_name} is not defined. Defined are {defined}" - - if "dry_run" in args and args["dry_run"]: - return error or f"Dry run: Would be calling {fn_name} with {args}." - - if error: - raise PyTransifexException(error) - - try: - return getattr(self, fn_name)(**args) - except Exception as error: - return str(error) - def create_project( self, project_slug: str, diff --git a/pytransifex/cli.py b/pytransifex/cli.py index 3b8693a..8dad1e4 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -1,7 +1,7 @@ from typing import Any import click -from pytransifex.api import Transifex +from pytransifex.api_old import Transifex from pytransifex.config import Config @@ -9,9 +9,6 @@ def format_args(args: dict[str, Any]) -> dict[str, Any]: return {k.replace("-", "_"): v for k, v in args.items()} -import click - - @click.group() def cli(): pass @@ -19,7 +16,7 @@ def cli(): @click.option("--verbose", is_flag=True, default=False) @click.argument("file_name", required=True) -@cli.command("push", help="Push documentation") +@cli.command("push", help="Push translation strings") def push(file_name: str, verbose: bool): click.echo(f"push: {file_name}") if verbose: @@ -28,7 +25,7 @@ def push(file_name: str, verbose: bool): @click.option("--only-lang", "-l", default="all") @click.argument("dir_name", required=True) -@cli.command("pull", help="Pull documentation") +@cli.command("pull", help="Pull translation strings") def pull(dir_name: str, only_lang: str): click.echo(f"pull: {dir_name}, {only_lang}") diff --git a/pytransifex/config.py b/pytransifex/config.py index 5bc53e4..8a914ad 100644 --- a/pytransifex/config.py +++ b/pytransifex/config.py @@ -1,5 +1,6 @@ from os import environ from typing import NamedTuple +from dotenv import load_dotenv from pytransifex.exceptions import PyTransifexException @@ -7,16 +8,19 @@ class Config(NamedTuple): api_token: str organization: str i18n_type: str + host = "https://rest.api.transifex.com" @classmethod def from_env(cls) -> "Config": - token = environ.get("TX_API_TOKEN") + load_dotenv() + + token = environ.get("TX_TOKEN") organization = environ.get("ORGANIZATION") i18n_type = environ.get("I18N_TYPE", "PO") - if any(v is None for v in [token, organization]): + if any(not v for v in [token, organization, i18n_type]): raise PyTransifexException( - "Both 'TX_API_TOKEN' and 'ORGANIZATION' must be set environment variables. Aborting now." + "Envars 'TX_TOKEN' and 'ORGANIZATION' must be set to non-empty values. Aborting now." ) return cls(token, organization, i18n_type) # type: ignore diff --git a/pytransifex/interfaces.py b/pytransifex/interfaces.py index a7033e6..0365d9d 100644 --- a/pytransifex/interfaces.py +++ b/pytransifex/interfaces.py @@ -1,23 +1,16 @@ from typing import Any -from abc import abstractclassmethod, abstractmethod, ABC +from abc import abstractmethod, ABC -from pytransifex.config import Config - -class IsTranslator(ABC): +class Tx(ABC): # TODO - # This interface satisfies the expectations of qgis-plugin-cli - # but it falls short of requiring an implementation for methods - # that qgis-plugin-cli does not use: + # This interface modelled after api.py:Transifex satisfies the expectations of qgis-plugin-cli + # but it falls short of requiring an implementation for methods that qgis-plugin-cli does not use: # coordinator, create_translation, delete_project, delete_resource, delete_team # The question is whether I should implement them. Is there's a consumer downstream? - @abstractclassmethod - def list_funcs(cls) -> list[str]: - raise NotImplementedError - @abstractmethod - def exec(self, fn_name: str, args: dict[str, Any]) -> Any: + def login(self): raise NotImplementedError @abstractmethod diff --git a/pytransifex/utils.py b/pytransifex/utils.py index 31fa0df..d45274c 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -1,7 +1,7 @@ from functools import wraps -def auth_client(f): +def ensure_logged_client(f): @wraps(f) def capture_args(instance, *args, **kwargs): if not instance.logged_in: diff --git a/requirements-locked.txt b/requirements-locked.txt index 961b398..f63d0d8 100644 --- a/requirements-locked.txt +++ b/requirements-locked.txt @@ -44,6 +44,8 @@ pyparsing==3.0.9 # via packaging pyseeyou==1.0.2 # via transifex-python +python-dotenv==0.21.0 + # via -r requirements.txt pytz==2022.5 # via # babel diff --git a/requirements.txt b/requirements.txt index 8601b34..c799984 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ sphinx sphinx_rtd_theme nose2 transifex-python -click \ No newline at end of file +click +python-dotenv \ No newline at end of file diff --git a/setup.py b/setup.py index 442a4df..b1ca7c3 100644 --- a/setup.py +++ b/setup.py @@ -1,35 +1,35 @@ from setuptools import setup import sys -python_min_version = (3, 6) +python_min_version = (3, 10) if sys.version_info < python_min_version: - sys.exit('pytransifex requires at least Python version {vmaj}.{vmin}.\n' - 'You are currently running this installation with\n\n{curver}'.format( - vmaj=python_min_version[0], - vmin=python_min_version[1], - curver=sys.version)) + sys.exit( + "pytransifex requires at least Python version {vmaj}.{vmin}.\n" + "You are currently running this installation with\n\n{curver}".format( + vmaj=python_min_version[0], vmin=python_min_version[1], curver=sys.version + ) + ) setup( - name = 'pytransifex', - packages = [ - 'pytransifex' + name="pytransifex", + packages=["pytransifex"], + version="[VERSION]", + description="Yet another Python Transifex API.", + author="Denis Rouzaud", + author_email="denis.rouzaud@gmail.com", + url="https://github.com/opengisch/pytransifex", + download_url="https://github.com/opengisch/pytransifex/archive/[VERSION].tar.gz", # I'll explain this in a second + keywords=["Transifex"], + classifiers=[ + "Topic :: Software Development :: Localization", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Intended Audience :: System Administrators", + "Development Status :: 3 - Alpha", ], - version = '[VERSION]', - description = 'Yet another Python Transifex API.', - author = 'Denis Rouzaud', - author_email = 'denis.rouzaud@gmail.com', - url = 'https://github.com/opengisch/pytransifex', - download_url = 'https://github.com/opengisch/pytransifex/archive/[VERSION].tar.gz', # I'll explain this in a second - keywords = ['Transifex'], - classifiers = [ - 'Topic :: Software Development :: Localization', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', - 'Intended Audience :: System Administrators', - 'Development Status :: 3 - Alpha' - ], - install_requires = [ - 'requests' - ], - python_requires=">={vmaj}.{vmin}".format(vmaj=python_min_version[0], vmin=python_min_version[1]), + install_requires=["requests", "click"], + python_requires=">={vmaj}.{vmin}".format( + vmaj=python_min_version[0], vmin=python_min_version[1] + ), + entry_points={"console_scripts": ["pytx = pytransifex.cli:cli"]}, ) diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 96fbaec..e9bda34 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -1,23 +1,84 @@ import unittest from pytransifex.config import Config -from pytransifex.api import Transifex as TOld -from pytransifex.api_new import Transifex as TNew -from pytransifex.interfaces import IsTranslator - - -config = Config("tok", "orga", "ln") +from pytransifex.api_new import Transifex +from pytransifex.interfaces import Tx +from transifex.api.jsonapi.exceptions import JsonApiException +from transifex.api.jsonapi.resources import Resource class TestNewApi(unittest.TestCase): - def test_old_api_satisfies_protocol(self): - old_api = TOld(config) - assert isinstance(old_api, IsTranslator) + def setUp(self): + config = Config.from_env() + self.tx = Transifex(config, defer_login=True) + self.project_slug = "test_project_pytransifex" + self.project_name = "Test Project PyTransifex" + self.resource_slug = "test_resource_fr" + self.resource_name = "Test Resource FR" + self.path_to_file = "./test_resource_fr.po" def test_new_api_satisfies_protocol(self): - new_api = TNew(config, defer_login=True) - assert isinstance(new_api, IsTranslator) + assert isinstance(self.tx, Tx) + + def test_create_project(self): + resource: None | Resource = None + try: + resource = self.tx.create_project( + project_name=self.project_name, + project_slug=self.project_slug, + private=True, + ) + assert True + + except JsonApiException as error: + if "already exists" in error.detail: + if project := self.tx.get_project(project_slug=self.project_slug): + project.delete() + + finally: + if resource: + resource.delete() + + def test_list_resources(self): + resources = self.tx.list_resources(project_slug=self.project_slug) + assert resources is not None + + def test_create_resource(self): + self.tx.create_resource( + project_slug=self.project_slug, + resource_name=self.resource_name, + resource_slug=self.resource_slug, + path_to_file=self.path_to_file, + ) + assert True + + +""" + def test_update_source_translation(self): + self.update_source_translation( + project_slug=self.project_slug, + path_to_file=self.path_to_file, + resource_slug=self.resource_slug, + ) + assert True + + def test_get_translation(self): + pass + + def test_list_languages(self): + languages = self.list_languages(project_slug=self.project_slug) + assert languages + + def test_create_language(self): + pass + + def test_project_exists(self): + verdict = self.project_exists(project_slug=self.project_slug) + assert verdict + def test_ping(self): + pass +""" if __name__ == "__main__": unittest.main() diff --git a/tests/test_api_old.py b/tests/test_api_old.py index 87fa3f2..93cc502 100644 --- a/tests/test_api_old.py +++ b/tests/test_api_old.py @@ -5,7 +5,7 @@ from pytransifex.config import Config from pytransifex.exceptions import PyTransifexException -from pytransifex.api import Transifex +from pytransifex.api_old import Transifex class TestTranslation(unittest.TestCase): diff --git a/tests/test_cli.py b/tests/test_cli.py index 9ca4db2..ebe1dc2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,7 +3,7 @@ from pytransifex.config import Config from pytransifex.cli import cli -from pytransifex.api import Transifex +from pytransifex.api_old import Transifex class TestCli(unittest.TestCase): diff --git a/tests/test_resource_fr.po b/tests/test_resource_fr.po new file mode 100644 index 0000000..c57eff5 --- /dev/null +++ b/tests/test_resource_fr.po @@ -0,0 +1 @@ +Hello World! \ No newline at end of file From 4b3780898535eb1c7137fe4605d6626344dce2f4 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 1 Nov 2022 11:30:22 +0100 Subject: [PATCH 04/34] Better error handling. --- pytransifex/api_new.py | 78 ++++++++++++++++++++++----------------- pytransifex/config.py | 3 +- pytransifex/interfaces.py | 4 -- pytransifex/utils.py | 2 +- tests/test_api_new.py | 34 ++++++----------- 5 files changed, 59 insertions(+), 62 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 038970a..954f32c 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,17 +1,17 @@ from typing import Any from transifex.api import transifex_api as tx_api from transifex.api.jsonapi.resources import Resource +from transifex.api.jsonapi import exceptions as json_api_exc from pytransifex.config import Config from pytransifex.interfaces import Tx -from pytransifex.utils import ensure_logged_client +from pytransifex.utils import ensure_login class Client(Tx): """ - The proper Transifex client expecte by the cli and other consumers. - By default instances are created and logged in 'lazyly' -- when creation and login cannot be deferred any longer. - Methods defined as "..." are left for further discussion; they are not used anywhere in qgis-plugin-cli. + The proper Transifex client expected by the cli and other consumers. + By default instances are created and logged in 'lazyly' -- when creation or login cannot be deferred any longer. """ def __init__(self, config: Config, defer_login: bool = False): @@ -36,7 +36,7 @@ def login(self): ) self.logged_in = True - @ensure_logged_client + @ensure_login def create_project( self, project_slug: str, @@ -58,19 +58,24 @@ def create_project( organization=organization, ) - @ensure_logged_client - def get_project(self, project_slug: str) -> Resource: + @ensure_login + def get_project(self, project_slug: str) -> None | Resource: if projects := self._organization_api_object.fetch("projects"): - return projects.get(slug=project_slug) - raise Exception(f"Project not found: {project_slug}") - - @ensure_logged_client - def list_resources(self, project_slug: str) -> list[Any]: + try: + return projects.get(slug=project_slug) + except json_api_exc.DoesNotExist: + return None + raise Exception(f"Unable to find any project with thus slug: '{project_slug}'") + + @ensure_login + def list_resources(self, project_slug: str) -> list[Resource]: if projects := self._organization_api_object.fetch("projects"): return projects.filter(slug=project_slug) - raise Exception(f"Project not found {project_slug}") + raise Exception( + f"Unable to find any project under this organization: '{self.organization}'" + ) - @ensure_logged_client + @ensure_login def create_resource( self, project_slug: str, @@ -87,52 +92,59 @@ def create_resource( file_handler = open(path_to_file, "r") content = file_handler.read() file_handler.close() - self.api.ResourceStringsAsyncUpload.upload(resource, content) - @ensure_logged_client + @ensure_login def update_source_translation( self, project_slug: str, resource_slug: str, path_to_file: str ): - resource = self.api.Resource.get(slug=resource_slug, project_slug=project_slug) - file_handler = open(path_to_file, "r") - content = file_handler.read() - file_handler.close() - self.api.ResourceStringsAsyncUpload(resource, content) - - @ensure_logged_client + if resource := self.api.Resource.get( + slug=resource_slug, project_slug=project_slug + ): + file_handler = open(path_to_file, "r") + content = file_handler.read() + file_handler.close() + self.api.ResourceStringsAsyncUpload(resource, content) + else: + raise Exception( + f"Unable to find resource '{resource_slug}' in project '{project_slug}'" + ) + + @ensure_login def get_translation( self, project_slug: str, resource_slug: str, language: str, path_to_file: str ): - ... + pass - @ensure_logged_client + @ensure_login def list_languages(self, project_slug: str) -> list[Any]: if projects := self._organization_api_object.fetch("projects"): if project := projects.get(slug=project_slug): return project.fetch("languages") - raise Exception(f"Unable to find any data for this project {project_slug}") + raise Exception( + f"Unable to find any project with this slug: '{project_slug}'" + ) raise Exception( - f"You need at least 1 project to be able to retrieve a list of projects." + f"Unable to find any project under this organization: '{self.organization}'" ) - @ensure_logged_client + @ensure_login def create_language(self, project_slug: str, path_to_file: str, resource_slug: str): - ... + pass - @ensure_logged_client + @ensure_login def project_exists(self, project_slug: str) -> bool: if projects := self._organization_api_object.fetch("projects"): if projects.get(slug=project_slug): return True return False raise Exception( - f"No project could be found under this organization: {self.organization}" + f"No project could be found under this organization: '{self.organization}'" ) - @ensure_logged_client + @ensure_login def ping(self): - ... + pass class Transifex: diff --git a/pytransifex/config.py b/pytransifex/config.py index 8a914ad..2f71552 100644 --- a/pytransifex/config.py +++ b/pytransifex/config.py @@ -1,7 +1,6 @@ from os import environ from typing import NamedTuple from dotenv import load_dotenv -from pytransifex.exceptions import PyTransifexException class Config(NamedTuple): @@ -19,7 +18,7 @@ def from_env(cls) -> "Config": i18n_type = environ.get("I18N_TYPE", "PO") if any(not v for v in [token, organization, i18n_type]): - raise PyTransifexException( + raise Exception( "Envars 'TX_TOKEN' and 'ORGANIZATION' must be set to non-empty values. Aborting now." ) diff --git a/pytransifex/interfaces.py b/pytransifex/interfaces.py index 0365d9d..7f8c61a 100644 --- a/pytransifex/interfaces.py +++ b/pytransifex/interfaces.py @@ -9,10 +9,6 @@ class Tx(ABC): # coordinator, create_translation, delete_project, delete_resource, delete_team # The question is whether I should implement them. Is there's a consumer downstream? - @abstractmethod - def login(self): - raise NotImplementedError - @abstractmethod def create_project( self, diff --git a/pytransifex/utils.py b/pytransifex/utils.py index d45274c..e486eb2 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -1,7 +1,7 @@ from functools import wraps -def ensure_logged_client(f): +def ensure_login(f): @wraps(f) def capture_args(instance, *args, **kwargs): if not instance.logged_in: diff --git a/tests/test_api_new.py b/tests/test_api_new.py index e9bda34..176ab85 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -3,8 +3,6 @@ from pytransifex.config import Config from pytransifex.api_new import Transifex from pytransifex.interfaces import Tx -from transifex.api.jsonapi.exceptions import JsonApiException -from transifex.api.jsonapi.resources import Resource class TestNewApi(unittest.TestCase): @@ -17,31 +15,23 @@ def setUp(self): self.resource_name = "Test Resource FR" self.path_to_file = "./test_resource_fr.po" - def test_new_api_satisfies_protocol(self): + if project := self.tx.get_project(project_slug=self.project_slug): + project.delete() + + def test_new_api_satisfies_abc(self): assert isinstance(self.tx, Tx) def test_create_project(self): - resource: None | Resource = None - try: - resource = self.tx.create_project( - project_name=self.project_name, - project_slug=self.project_slug, - private=True, - ) - assert True - - except JsonApiException as error: - if "already exists" in error.detail: - if project := self.tx.get_project(project_slug=self.project_slug): - project.delete() - - finally: - if resource: - resource.delete() + _ = self.tx.create_project( + project_name=self.project_name, + project_slug=self.project_slug, + private=True, + ) + assert True def test_list_resources(self): - resources = self.tx.list_resources(project_slug=self.project_slug) - assert resources is not None + _ = self.tx.list_resources(project_slug=self.project_slug) + assert True def test_create_resource(self): self.tx.create_resource( From b86857ccbb48e2d05b663b79a0119f8bcf3c4a0e Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 1 Nov 2022 12:42:00 +0100 Subject: [PATCH 05/34] Fixed uploading failure. --- pytransifex/api_new.py | 30 +++++++++++++++++----------- tests/test_api_new.py | 41 ++++++++++++++++++++++++--------------- tests/test_resource_fr.po | 27 +++++++++++++++++++++++++- 3 files changed, 70 insertions(+), 28 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 954f32c..0270dbd 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,3 +1,4 @@ +from distutils.command.upload import upload from typing import Any from transifex.api import transifex_api as tx_api from transifex.api.jsonapi.resources import Resource @@ -65,7 +66,6 @@ def get_project(self, project_slug: str) -> None | Resource: return projects.get(slug=project_slug) except json_api_exc.DoesNotExist: return None - raise Exception(f"Unable to find any project with thus slug: '{project_slug}'") @ensure_login def list_resources(self, project_slug: str) -> list[Resource]: @@ -86,13 +86,22 @@ def create_resource( if not (resource_slug or resource_name): raise Exception("Please give either a resource_slug or resource_name") - resource = self.api.Resource.create(name=resource_name, slug=resource_slug) - resource.save(project=project_slug) + if project := self.get_project(project_slug=project_slug): + resource = self.api.Resource.create( + project=project, + name=resource_name, + slug=resource_slug, + i18n_format=tx_api.I18nFormat(id=self.i18n_type), + ) + + with open(path_to_file, "r") as fh: + content = fh.read() + self.api.ResourceStringsAsyncUpload.upload(content, resource=resource) - file_handler = open(path_to_file, "r") - content = file_handler.read() - file_handler.close() - self.api.ResourceStringsAsyncUpload.upload(resource, content) + else: + raise Exception( + f"Not project could be found wiht the slug '{project_slug}'. Please create a project first." + ) @ensure_login def update_source_translation( @@ -101,10 +110,9 @@ def update_source_translation( if resource := self.api.Resource.get( slug=resource_slug, project_slug=project_slug ): - file_handler = open(path_to_file, "r") - content = file_handler.read() - file_handler.close() - self.api.ResourceStringsAsyncUpload(resource, content) + with open(path_to_file, "r") as fh: + content = fh.read() + self.api.ResourceStringsAsyncUpload(resource, content) else: raise Exception( f"Unable to find resource '{resource_slug}' in project '{project_slug}'" diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 176ab85..5110694 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -1,10 +1,20 @@ import unittest +from pathlib import Path from pytransifex.config import Config from pytransifex.api_new import Transifex from pytransifex.interfaces import Tx +def ensure_project(instance): + if not instance.tx.get_project(project_slug=instance.project_slug): + instance.tx.create_project( + project_name=instance.project_name, + project_slug=instance.project_slug, + private=True, + ) + + class TestNewApi(unittest.TestCase): def setUp(self): config = Config.from_env() @@ -13,27 +23,26 @@ def setUp(self): self.project_name = "Test Project PyTransifex" self.resource_slug = "test_resource_fr" self.resource_name = "Test Resource FR" - self.path_to_file = "./test_resource_fr.po" + self.path_to_file = Path(Path.cwd()).joinpath("tests", "test_resource_fr.po") if project := self.tx.get_project(project_slug=self.project_slug): + print("Found a project. Deleting...") project.delete() - def test_new_api_satisfies_abc(self): + def test1_new_api_satisfies_abc(self): assert isinstance(self.tx, Tx) - def test_create_project(self): - _ = self.tx.create_project( - project_name=self.project_name, - project_slug=self.project_slug, - private=True, - ) + def test2_create_project(self): + ensure_project(self) assert True - def test_list_resources(self): + def test3_list_resources(self): + ensure_project(self) _ = self.tx.list_resources(project_slug=self.project_slug) assert True - def test_create_resource(self): + def test4_create_resource(self): + ensure_project(self) self.tx.create_resource( project_slug=self.project_slug, resource_name=self.resource_name, @@ -44,7 +53,7 @@ def test_create_resource(self): """ - def test_update_source_translation(self): + def test5_update_source_translation(self): self.update_source_translation( project_slug=self.project_slug, path_to_file=self.path_to_file, @@ -52,21 +61,21 @@ def test_update_source_translation(self): ) assert True - def test_get_translation(self): + def test6_get_translation(self): pass - def test_list_languages(self): + def test7_list_languages(self): languages = self.list_languages(project_slug=self.project_slug) assert languages - def test_create_language(self): + def test8_create_language(self): pass - def test_project_exists(self): + def test9_project_exists(self): verdict = self.project_exists(project_slug=self.project_slug) assert verdict - def test_ping(self): + def test10_ping(self): pass """ diff --git a/tests/test_resource_fr.po b/tests/test_resource_fr.po index c57eff5..7704ba3 100644 --- a/tests/test_resource_fr.po +++ b/tests/test_resource_fr.po @@ -1 +1,26 @@ -Hello World! \ No newline at end of file +msgid "" +msgstr "" +"Project-Id-Version: Lingohub 1.0.1\n" +"Report-Msgid-Bugs-To: support@lingohub.com \n" +"Last-Translator: Marko Bošković \n" +"Language: de\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +msgid "Let’s make the web multilingual." +msgstr "Machen wir das Internet mehrsprachig." + +msgid "We connect developers and translators around the globe " +"on Lingohub for a fantastic localization experience." +msgstr "Wir verbinden Entwickler mit Übersetzern weltweit " +"auf Lingohub für ein fantastisches Lokalisierungs-Erlebnis." + +msgid "Welcome back, %1$s! Your last visit was on %2$s" +msgstr "Willkommen zurück, %1$s! Dein letzter Besuch war am %2$s" + +msgid "%d page read." +msgid_plural "%d pages read." +msgstr[0] "Eine Seite gelesen wurde." +msgstr[1] "%d Seiten gelesen wurden." \ No newline at end of file From 92315853710d5f1933f8d201cb34110ac14280d2 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 1 Nov 2022 16:49:48 +0100 Subject: [PATCH 06/34] Added remaining tests. --- pytransifex/api_new.py | 76 +++++++++++++++++++++++++++--------------- tests/test_api_new.py | 68 ++++++++++++++++++------------------- 2 files changed, 84 insertions(+), 60 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 0270dbd..aa577da 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,8 +1,8 @@ -from distutils.command.upload import upload from typing import Any +from pathlib import Path from transifex.api import transifex_api as tx_api from transifex.api.jsonapi.resources import Resource -from transifex.api.jsonapi import exceptions as json_api_exc +from transifex.api.jsonapi import exceptions as tx_exc from pytransifex.config import Config from pytransifex.interfaces import Tx @@ -46,25 +46,29 @@ def create_project( private: bool = False, *args, **kwargs, - ) -> Resource: + ) -> None | Resource: """Create a project. args, kwargs are there to absorb unnecessary arguments from consumers.""" source_language = self.api.Language.get(code=source_language_code) organization = self._organization_api_object - return self.api.Project.create( - name=project_name, - slug=project_slug, - source_language=source_language, - private=private, - organization=organization, - ) + try: + return self.api.Project.create( + name=project_name, + slug=project_slug, + source_language=source_language, + private=private, + organization=organization, + ) + except tx_exc.JsonApiException as error: + if "already exists" in error.detail: + return self.get_project(project_slug=project_slug) @ensure_login def get_project(self, project_slug: str) -> None | Resource: if projects := self._organization_api_object.fetch("projects"): try: return projects.get(slug=project_slug) - except json_api_exc.DoesNotExist: + except tx_exc.DoesNotExist: return None @ensure_login @@ -79,7 +83,7 @@ def list_resources(self, project_slug: str) -> list[Resource]: def create_resource( self, project_slug: str, - path_to_file: str, + path_to_file: Path | str, resource_slug: str | None = None, resource_name: str | None = None, ): @@ -105,24 +109,35 @@ def create_resource( @ensure_login def update_source_translation( - self, project_slug: str, resource_slug: str, path_to_file: str + self, project_slug: str, resource_slug: str, path_to_file: Path | str ): - if resource := self.api.Resource.get( - slug=resource_slug, project_slug=project_slug - ): - with open(path_to_file, "r") as fh: - content = fh.read() - self.api.ResourceStringsAsyncUpload(resource, content) - else: + if not "slug" in self._organization_api_object.attributes: raise Exception( - f"Unable to find resource '{resource_slug}' in project '{project_slug}'" + "Unable to fetch resource for this organization; define an 'organization slug' first." ) + if project := self.get_project(project_slug=project_slug): + if resources := project.fetch("resources"): + if resource := resources.get(slug=resource_slug): + with open(path_to_file, "r") as fh: + content = fh.read() + # FIXME + self.api.ResourceStringsAsyncUpload(content, resource=resource) + return + + raise Exception( + f"Unable to find resource '{resource_slug}' in project '{project_slug}'" + ) + @ensure_login def get_translation( - self, project_slug: str, resource_slug: str, language: str, path_to_file: str - ): - pass + self, + project_slug: str, + language: str, + *args, + **kwargs + ) -> list[str]: + return self.list_languages(project_slug=project_slug) @ensure_login def list_languages(self, project_slug: str) -> list[Any]: @@ -137,8 +152,17 @@ def list_languages(self, project_slug: str) -> list[Any]: ) @ensure_login - def create_language(self, project_slug: str, path_to_file: str, resource_slug: str): - pass + def create_language( + self, + project_slug: str, + language_code: str, + coordinators: None | list[Any] = None, + ): + if project := self.get_project(project_slug=project_slug): + project.add("languages", [self.api.Language.get(code=language_code)]) + + if coordinators: + project.add("coordinators", coordinators) @ensure_login def project_exists(self, project_slug: str) -> bool: diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 5110694..2756f95 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -6,43 +6,43 @@ from pytransifex.interfaces import Tx -def ensure_project(instance): - if not instance.tx.get_project(project_slug=instance.project_slug): - instance.tx.create_project( - project_name=instance.project_name, - project_slug=instance.project_slug, - private=True, - ) - - class TestNewApi(unittest.TestCase): - def setUp(self): + @classmethod + def setUpClass(cls): config = Config.from_env() - self.tx = Transifex(config, defer_login=True) - self.project_slug = "test_project_pytransifex" - self.project_name = "Test Project PyTransifex" - self.resource_slug = "test_resource_fr" - self.resource_name = "Test Resource FR" - self.path_to_file = Path(Path.cwd()).joinpath("tests", "test_resource_fr.po") - - if project := self.tx.get_project(project_slug=self.project_slug): - print("Found a project. Deleting...") + cls.tx = Transifex(config, defer_login=True) + cls.project_slug = "test_project_pytransifex" + cls.project_name = "Test Project PyTransifex" + cls.resource_slug = "test_resource_fr" + cls.resource_name = "Test Resource FR" + cls.path_to_file = Path(Path.cwd()).joinpath("tests", "test_resource_fr.po") + + if project := cls.tx.get_project(project_slug=cls.project_slug): + print("Found old project, removing.") + project.delete() + + print("Creating a brand new project") + cls.tx.create_project( + project_name=cls.project_name, project_slug=cls.project_slug, private=True + ) + + @classmethod + def tearDownClass(cls): + if project := cls.tx.get_project(project_slug=cls.project_slug): project.delete() def test1_new_api_satisfies_abc(self): assert isinstance(self.tx, Tx) def test2_create_project(self): - ensure_project(self) - assert True + # Done in setUpClass + pass def test3_list_resources(self): - ensure_project(self) _ = self.tx.list_resources(project_slug=self.project_slug) assert True def test4_create_resource(self): - ensure_project(self) self.tx.create_resource( project_slug=self.project_slug, resource_name=self.resource_name, @@ -51,10 +51,8 @@ def test4_create_resource(self): ) assert True - -""" def test5_update_source_translation(self): - self.update_source_translation( + self.tx.update_source_translation( project_slug=self.project_slug, path_to_file=self.path_to_file, resource_slug=self.resource_slug, @@ -62,22 +60,24 @@ def test5_update_source_translation(self): assert True def test6_get_translation(self): - pass + translation = self.tx.get_translation(project_slug=self.project_slug, language="fr_CH") + assert translation def test7_list_languages(self): - languages = self.list_languages(project_slug=self.project_slug) - assert languages + languages = self.tx.list_languages(project_slug=self.project_slug) + assert languages is not None def test8_create_language(self): - pass + self.tx.create_language(self.project_slug, "fr_CH") def test9_project_exists(self): - verdict = self.project_exists(project_slug=self.project_slug) - assert verdict + verdict = self.tx.project_exists(project_slug=self.project_slug) + assert verdict is not None def test10_ping(self): - pass -""" + self.tx.ping() + assert True + if __name__ == "__main__": unittest.main() From 476289962c5cbd46f30164b1fac1eaae2d09052a Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 3 Nov 2022 09:55:35 +0100 Subject: [PATCH 07/34] Fixing misunderstood method. --- pytransifex/api_new.py | 45 +++++++++++++++++++++++++++++++++++++----- tests/test_api_new.py | 7 ++++++- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index aa577da..4fe216d 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,3 +1,5 @@ +import requests + from typing import Any from pathlib import Path from transifex.api import transifex_api as tx_api @@ -65,6 +67,7 @@ def create_project( @ensure_login def get_project(self, project_slug: str) -> None | Resource: + """Fetches the project matching the given slug""" if projects := self._organization_api_object.fetch("projects"): try: return projects.get(slug=project_slug) @@ -73,6 +76,7 @@ def get_project(self, project_slug: str) -> None | Resource: @ensure_login def list_resources(self, project_slug: str) -> list[Resource]: + """List all resources for the project passed as argument""" if projects := self._organization_api_object.fetch("projects"): return projects.filter(slug=project_slug) raise Exception( @@ -87,6 +91,7 @@ def create_resource( resource_slug: str | None = None, resource_name: str | None = None, ): + """Create a resource using the given file contents, slugs and names""" if not (resource_slug or resource_name): raise Exception("Please give either a resource_slug or resource_name") @@ -111,6 +116,10 @@ def create_resource( def update_source_translation( self, project_slug: str, resource_slug: str, path_to_file: Path | str ): + """ + Update the translation strings for the given resource using the content of the file + passsed as argument + """ if not "slug" in self._organization_api_object.attributes: raise Exception( "Unable to fetch resource for this organization; define an 'organization slug' first." @@ -133,14 +142,34 @@ def update_source_translation( def get_translation( self, project_slug: str, - language: str, - *args, - **kwargs - ) -> list[str]: - return self.list_languages(project_slug=project_slug) + resource_slug, + language_code: str, + path_to_file: Path | str, + ): + """Fetch the resources matching the language given as parameter for the project""" + if project := self.get_project(project_slug=project_slug): + if resource := project.get(resource_slug=resource_slug): + url = self.api.ResourceTranslationsAsyncDownload.download( + resource=resource, language=language_code + ) + translated_content = requests.get(url).text + with open(path_to_file, "w") as fh: + fh.write(translated_content) + else: + raise Exception( + f"Unable to find any resource with this slug: '{resource_slug}'" + ) + else: + raise Exception( + f"Couldn't find any project with this slug: '{project_slug}'" + ) @ensure_login def list_languages(self, project_slug: str) -> list[Any]: + """ + List all languages for which there is at least 1 resource registered + under the parameterised project + """ if projects := self._organization_api_object.fetch("projects"): if project := projects.get(slug=project_slug): return project.fetch("languages") @@ -158,6 +187,7 @@ def create_language( language_code: str, coordinators: None | list[Any] = None, ): + """Create a new language resource in the remote Transifex repository""" if project := self.get_project(project_slug=project_slug): project.add("languages", [self.api.Language.get(code=language_code)]) @@ -166,6 +196,7 @@ def create_language( @ensure_login def project_exists(self, project_slug: str) -> bool: + """Check if the project exists in the remote Transifex repository""" if projects := self._organization_api_object.fetch("projects"): if projects.get(slug=project_slug): return True @@ -176,6 +207,10 @@ def project_exists(self, project_slug: str) -> bool: @ensure_login def ping(self): + """ + Exposing this just for the sake of satisfying qgis-plugin-cli's expectations + There is no need to ping the server on the current implementation, as connection is handled by the SDK + """ pass diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 2756f95..d8a3bbc 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -60,7 +60,12 @@ def test5_update_source_translation(self): assert True def test6_get_translation(self): - translation = self.tx.get_translation(project_slug=self.project_slug, language="fr_CH") + translation = self.tx.get_translation( + project_slug=self.project_slug, + resource_slug=self.resource_slug, + language_code="fr_CH", + path_to_file=self.path_to_file, + ) assert translation def test7_list_languages(self): From 02209288bd7a62f48e7e3dd6099f8d7ffcccddc0 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 3 Nov 2022 12:16:19 +0100 Subject: [PATCH 08/34] All tests OK --- pytransifex/api_new.py | 34 +++++++++++++++++++++------------- tests/test_api_new.py | 5 ----- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 4fe216d..516f7d5 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -87,7 +87,7 @@ def list_resources(self, project_slug: str) -> list[Resource]: def create_resource( self, project_slug: str, - path_to_file: Path | str, + path_to_file: Path, resource_slug: str | None = None, resource_name: str | None = None, ): @@ -114,7 +114,7 @@ def create_resource( @ensure_login def update_source_translation( - self, project_slug: str, resource_slug: str, path_to_file: Path | str + self, project_slug: str, resource_slug: str, path_to_file: Path ): """ Update the translation strings for the given resource using the content of the file @@ -130,8 +130,9 @@ def update_source_translation( if resource := resources.get(slug=resource_slug): with open(path_to_file, "r") as fh: content = fh.read() - # FIXME - self.api.ResourceStringsAsyncUpload(content, resource=resource) + self.api.ResourceStringsAsyncUpload.upload( + content, resource=resource + ) return raise Exception( @@ -144,20 +145,27 @@ def get_translation( project_slug: str, resource_slug, language_code: str, - path_to_file: Path | str, + path_to_file: Path, ): """Fetch the resources matching the language given as parameter for the project""" + # resource_id = f"o:{self.organization}:p:{project_slug}:r:{resource_slug}" + language = self.api.Language.get(code=language_code) if project := self.get_project(project_slug=project_slug): - if resource := project.get(resource_slug=resource_slug): - url = self.api.ResourceTranslationsAsyncDownload.download( - resource=resource, language=language_code - ) - translated_content = requests.get(url).text - with open(path_to_file, "w") as fh: - fh.write(translated_content) + if resources := project.fetch("resources"): + if resource := resources.get(slug=resource_slug): + url = self.api.ResourceTranslationsAsyncDownload.download( + resource=resource, language=language + ) + translated_content = requests.get(url).text + with open(path_to_file, "w") as fh: + fh.write(translated_content) + else: + raise Exception( + f"Unable to find any resource with this slug: '{resource_slug}'" + ) else: raise Exception( - f"Unable to find any resource with this slug: '{resource_slug}'" + f"Unable to find any resource for this project: '{project_slug}'" ) else: raise Exception( diff --git a/tests/test_api_new.py b/tests/test_api_new.py index d8a3bbc..db0e02d 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -26,11 +26,6 @@ def setUpClass(cls): project_name=cls.project_name, project_slug=cls.project_slug, private=True ) - @classmethod - def tearDownClass(cls): - if project := cls.tx.get_project(project_slug=cls.project_slug): - project.delete() - def test1_new_api_satisfies_abc(self): assert isinstance(self.tx, Tx) From 4085e48e25442a7687c6aa00095f03f98760ded7 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Fri, 4 Nov 2022 12:25:19 +0100 Subject: [PATCH 09/34] map_async, aiofiles --- pytransifex/api_new.py | 92 ++++++++++++++++++++++++++++------------- pytransifex/cli.py | 11 +++-- pytransifex/config.py | 4 +- pytransifex/utils.py | 42 +++++++++++++++++++ requirements-locked.txt | 28 ++++++++++++- requirements.txt | 4 +- tests/test_api_new.py | 20 +++++++-- tests/test_utils.py | 37 +++++++++++++++++ 8 files changed, 198 insertions(+), 40 deletions(-) create mode 100644 tests/test_utils.py diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 516f7d5..0741a07 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -4,11 +4,12 @@ from pathlib import Path from transifex.api import transifex_api as tx_api from transifex.api.jsonapi.resources import Resource -from transifex.api.jsonapi import exceptions as tx_exc +from transifex.api.jsonapi import JsonApiException +from transifex.api.jsonapi.exceptions import DoesNotExist from pytransifex.config import Config from pytransifex.interfaces import Tx -from pytransifex.utils import ensure_login +from pytransifex.utils import ensure_login, map_async class Client(Tx): @@ -20,8 +21,8 @@ class Client(Tx): def __init__(self, config: Config, defer_login: bool = False): """Extract config values, consumes API token against SDK client""" self.api_token = config.api_token - self.host = config.host - self.organization = config.organization + self.host = config.host_name + self.organization_name = config.organization_name self.i18n_type = config.i18n_type self.logged_in = False self.api = tx_api @@ -33,12 +34,15 @@ def login(self): if self.logged_in: return + # Authentication self.api.setup(host=self.host, auth=self.api_token) - self._organization_api_object = self.api.Organization.get( - slug=self.organization - ) self.logged_in = True + # Saving organization and projects to avoid round-trips + organization = self.api.Organization.get(slug=self.organization_name) + self.projects = organization.fetch("projects") + self.organization = organization + @ensure_login def create_project( self, @@ -51,7 +55,6 @@ def create_project( ) -> None | Resource: """Create a project. args, kwargs are there to absorb unnecessary arguments from consumers.""" source_language = self.api.Language.get(code=source_language_code) - organization = self._organization_api_object try: return self.api.Project.create( @@ -59,26 +62,26 @@ def create_project( slug=project_slug, source_language=source_language, private=private, - organization=organization, + organization=self.organization, ) - except tx_exc.JsonApiException as error: + except JsonApiException as error: if "already exists" in error.detail: return self.get_project(project_slug=project_slug) @ensure_login def get_project(self, project_slug: str) -> None | Resource: """Fetches the project matching the given slug""" - if projects := self._organization_api_object.fetch("projects"): + if self.projects: try: - return projects.get(slug=project_slug) - except tx_exc.DoesNotExist: + return self.projects.get(slug=project_slug) + except DoesNotExist: return None @ensure_login def list_resources(self, project_slug: str) -> list[Resource]: """List all resources for the project passed as argument""" - if projects := self._organization_api_object.fetch("projects"): - return projects.filter(slug=project_slug) + if self.projects: + return self.projects.filter(slug=project_slug) raise Exception( f"Unable to find any project under this organization: '{self.organization}'" ) @@ -120,7 +123,7 @@ def update_source_translation( Update the translation strings for the given resource using the content of the file passsed as argument """ - if not "slug" in self._organization_api_object.attributes: + if not "slug" in self.organization.attributes: raise Exception( "Unable to fetch resource for this organization; define an 'organization slug' first." ) @@ -148,7 +151,6 @@ def get_translation( path_to_file: Path, ): """Fetch the resources matching the language given as parameter for the project""" - # resource_id = f"o:{self.organization}:p:{project_slug}:r:{resource_slug}" language = self.api.Language.get(code=language_code) if project := self.get_project(project_slug=project_slug): if resources := project.fetch("resources"): @@ -178,8 +180,8 @@ def list_languages(self, project_slug: str) -> list[Any]: List all languages for which there is at least 1 resource registered under the parameterised project """ - if projects := self._organization_api_object.fetch("projects"): - if project := projects.get(slug=project_slug): + if self.projects: + if project := self.projects.get(slug=project_slug): return project.fetch("languages") raise Exception( f"Unable to find any project with this slug: '{project_slug}'" @@ -205,8 +207,8 @@ def create_language( @ensure_login def project_exists(self, project_slug: str) -> bool: """Check if the project exists in the remote Transifex repository""" - if projects := self._organization_api_object.fetch("projects"): - if projects.get(slug=project_slug): + if self.projects: + if self.projects.get(slug=project_slug): return True return False raise Exception( @@ -221,6 +223,45 @@ def ping(self): """ pass + @ensure_login + def get_translation_stats(self, project_slug: str) -> dict[str, Any]: + if self.projects: + if project := self.projects.get(slug=project_slug): + return self.api.ResourceLanguageStats(project=project).to_dict() + raise Exception(f"Unable to find translation for this project {project_slug}") + + @ensure_login + def push( + self, project_slug: str, resource_slugs: list[str], path_to_files: list[str] + ): + args = [ + tuple([pro, res, path]) + for pro, res, path in zip([project_slug], resource_slugs, path_to_files) + ] + map_async( + fn=self.update_source_translation, + args=args, + ) + + @ensure_login + def pull( + self, + project_slug: str, + resource_slugs: list[str], + language_codes: list[str], + path_to_files: list[str], + ): + args = [ + tuple([pro, res, lco, path]) + for pro, res, lco, path in zip( + [project_slug], resource_slugs, language_codes, path_to_files + ) + ] + map_async( + fn=self.get_translation, + args=args, + ) + class Transifex: """ @@ -230,12 +271,7 @@ class Transifex: client = None - def __new__(cls, config: Config | None = None, defer_login: bool = False): + def __new__(cls, defer_login: bool = False): if not cls.client: - - if not config: - raise Exception("Need to pass config") - - cls.client = Client(config, defer_login) - + cls.client = Client(Config.from_env(), defer_login) return cls.client diff --git a/pytransifex/cli.py b/pytransifex/cli.py index 8dad1e4..002643b 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -1,8 +1,9 @@ from typing import Any import click -from pytransifex.api_old import Transifex -from pytransifex.config import Config +from pytransifex.api_new import Transifex + +client = Transifex(defer_login=True) def format_args(args: dict[str, Any]) -> dict[str, Any]: @@ -18,7 +19,8 @@ def cli(): @click.argument("file_name", required=True) @cli.command("push", help="Push translation strings") def push(file_name: str, verbose: bool): - click.echo(f"push: {file_name}") + click.echo(f"Pushing: {file_name}") + client.push() if verbose: click.echo("Was it verbose enough?") @@ -27,7 +29,8 @@ def push(file_name: str, verbose: bool): @click.argument("dir_name", required=True) @cli.command("pull", help="Pull translation strings") def pull(dir_name: str, only_lang: str): - click.echo(f"pull: {dir_name}, {only_lang}") + client.pull() + click.echo(f"Pulling: {dir_name}, {only_lang}") if __name__ == "__main__": diff --git a/pytransifex/config.py b/pytransifex/config.py index 2f71552..1e62d32 100644 --- a/pytransifex/config.py +++ b/pytransifex/config.py @@ -5,9 +5,9 @@ class Config(NamedTuple): api_token: str - organization: str + organization_name: str i18n_type: str - host = "https://rest.api.transifex.com" + host_name = "https://rest.api.transifex.com" @classmethod def from_env(cls) -> "Config": diff --git a/pytransifex/utils.py b/pytransifex/utils.py index e486eb2..6f697aa 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -1,4 +1,10 @@ +from pathlib import Path +from typing import Any, Awaitable, Callable, Iterable, TypeAlias +from asyncio import run, gather, get_running_loop +from aiofiles import open +from aiohttp import ClientSession from functools import wraps +from inspect import iscoroutinefunction def ensure_login(f): @@ -9,3 +15,39 @@ def capture_args(instance, *args, **kwargs): return f(instance, *args, **kwargs) return capture_args + + + +def map_async( + *, + fn: Callable | None = None, + args: Iterable[Any] | None = None, + partials: Iterable[Any] | None = None, +) -> Iterable[Any]: + async def closure() -> Iterable[Any]: + tasks = [] + if iscoroutinefunction(fn) and args: + tasks = [fn(*a) for a in args] + else: + loop = get_running_loop() + if partials: + tasks = [loop.run_in_executor(None, p) for p in partials] + elif fn and args: + tasks = [loop.run_in_executor(None, fn, *a) for a in args] + return await gather(*tasks) + + return run(closure()) + + +async def write_file_async(path: Path, contents: str) -> None: + async with open(path, "w") as fh: + await fh.write(contents) + + +async def get_async_from_url(urls: list[str]) -> list[str]: + async def get_text(session: ClientSession, url: str) -> str: + response = await session.get(url) + return await response.text() + + async with ClientSession() as session: + return await gather(*[get_text(session, url) for url in urls]) diff --git a/requirements-locked.txt b/requirements-locked.txt index f63d0d8..490b8b1 100644 --- a/requirements-locked.txt +++ b/requirements-locked.txt @@ -4,16 +4,28 @@ # # pip-compile --output-file=requirements-locked.txt requirements.txt # +aiofiles==22.1.0 + # via -r requirements.txt +aiohttp==3.8.3 + # via -r requirements.txt +aiosignal==1.2.0 + # via aiohttp alabaster==0.7.12 # via sphinx asttokens==2.0.8 # via transifex-python +async-timeout==4.0.2 + # via aiohttp +attrs==22.1.0 + # via aiohttp babel==2.10.3 # via sphinx certifi==2022.9.24 # via requests charset-normalizer==2.1.1 - # via requests + # via + # aiohttp + # requests click==8.1.3 # via # -r requirements.txt @@ -22,16 +34,26 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme +frozenlist==1.3.1 + # via + # aiohttp + # aiosignal future==0.18.2 # via pyseeyou idna==3.4 - # via requests + # via + # requests + # yarl imagesize==1.4.1 # via sphinx jinja2==3.1.2 # via sphinx markupsafe==2.1.1 # via jinja2 +multidict==6.0.2 + # via + # aiohttp + # yarl nose2==0.12.0 # via -r requirements.txt packaging==21.3 @@ -85,3 +107,5 @@ transifex-python==3.0.2 # via -r requirements.txt urllib3==1.26.12 # via requests +yarl==1.8.1 + # via aiohttp diff --git a/requirements.txt b/requirements.txt index c799984..3413022 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,6 @@ sphinx_rtd_theme nose2 transifex-python click -python-dotenv \ No newline at end of file +python-dotenv +aiohttp +aiofiles \ No newline at end of file diff --git a/tests/test_api_new.py b/tests/test_api_new.py index db0e02d..7b0f49f 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -1,7 +1,6 @@ import unittest from pathlib import Path -from pytransifex.config import Config from pytransifex.api_new import Transifex from pytransifex.interfaces import Tx @@ -9,8 +8,7 @@ class TestNewApi(unittest.TestCase): @classmethod def setUpClass(cls): - config = Config.from_env() - cls.tx = Transifex(config, defer_login=True) + cls.tx = Transifex(defer_login=True) cls.project_slug = "test_project_pytransifex" cls.project_name = "Test Project PyTransifex" cls.resource_slug = "test_resource_fr" @@ -29,6 +27,7 @@ def setUpClass(cls): def test1_new_api_satisfies_abc(self): assert isinstance(self.tx, Tx) + """ def test2_create_project(self): # Done in setUpClass pass @@ -77,6 +76,21 @@ def test9_project_exists(self): def test10_ping(self): self.tx.ping() assert True + """ + + def test11_stats(self): + stats = self.tx.get_translation_stats(project_slug=self.project_slug) + print(dir(stats)) + print(str(stats)) + assert stats + + """ + def test12_stats(self): + self.tx.get_translation_stats(project_slug=self.project_slug) + + def test13_stats(self): + self.tx.get_translation_stats(project_slug=self.project_slug) + """ if __name__ == "__main__": diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..43dedcb --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,37 @@ +import unittest +from pytransifex.utils import map_async +from functools import partial +from asyncio import sleep as asleep +from time import sleep as tsleep + + +async def coroutine_fn(a: int, b: int) -> int: + await asleep(a) + return a + b + + +def non_coroutine_fn(a: int, b: int) -> int: + tsleep(a) + return a + b + + +class TestUtils(unittest.TestCase): + @classmethod + def setUpClass(cls): + it = [1,2,3] + cls.it = it + cls.args = [tuple([a, b]) for a, b in zip(it, it)] + cls.partials = [partial(non_coroutine_fn, a, b) for a, b in zip(it, it)] + cls.res = [2, 4, 6] + + def test1_map_async(self): + res = map_async(fn=coroutine_fn, args=self.args) + assert res == self.res + + def test2_map_async(self): + res = map_async(partials=self.partials) + assert res == self.res + + def test_3_map_async(self): + res = map_async(fn=coroutine_fn, args=self.args) + assert res == self.res \ No newline at end of file From 6a34eb5f817b33756079d204406a8b5bbc66cc3b Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Fri, 4 Nov 2022 14:28:24 +0100 Subject: [PATCH 10/34] Cleaned up --- pytransifex/api_new.py | 69 +++++++++++++++++++++++++++-------------- pytransifex/utils.py | 12 ++++--- requirements-locked.txt | 28 ++--------------- requirements.txt | 4 +-- tests/test_utils.py | 2 +- 5 files changed, 58 insertions(+), 57 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 0741a07..6f4a520 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -25,7 +25,6 @@ def __init__(self, config: Config, defer_login: bool = False): self.organization_name = config.organization_name self.i18n_type = config.i18n_type self.logged_in = False - self.api = tx_api if not defer_login: self.login() @@ -35,13 +34,14 @@ def login(self): return # Authentication - self.api.setup(host=self.host, auth=self.api_token) + tx_api.setup(host=self.host, auth=self.api_token) self.logged_in = True # Saving organization and projects to avoid round-trips - organization = self.api.Organization.get(slug=self.organization_name) + organization = tx_api.Organization.get(slug=self.organization_name) self.projects = organization.fetch("projects") self.organization = organization + print(f"Logged in as organization: {self.organization_name}") @ensure_login def create_project( @@ -54,16 +54,18 @@ def create_project( **kwargs, ) -> None | Resource: """Create a project. args, kwargs are there to absorb unnecessary arguments from consumers.""" - source_language = self.api.Language.get(code=source_language_code) + source_language = tx_api.Language.get(code=source_language_code) try: - return self.api.Project.create( + res = tx_api.Project.create( name=project_name, slug=project_slug, source_language=source_language, private=private, organization=self.organization, ) + print("Project created!") + return res except JsonApiException as error: if "already exists" in error.detail: return self.get_project(project_slug=project_slug) @@ -73,7 +75,9 @@ def get_project(self, project_slug: str) -> None | Resource: """Fetches the project matching the given slug""" if self.projects: try: - return self.projects.get(slug=project_slug) + res = self.projects.get(slug=project_slug) + print("Got the project!") + return res except DoesNotExist: return None @@ -81,7 +85,9 @@ def get_project(self, project_slug: str) -> None | Resource: def list_resources(self, project_slug: str) -> list[Resource]: """List all resources for the project passed as argument""" if self.projects: - return self.projects.filter(slug=project_slug) + res = self.projects.filter(slug=project_slug) + print("Obtained these resources") + return res raise Exception( f"Unable to find any project under this organization: '{self.organization}'" ) @@ -99,7 +105,7 @@ def create_resource( raise Exception("Please give either a resource_slug or resource_name") if project := self.get_project(project_slug=project_slug): - resource = self.api.Resource.create( + resource = tx_api.Resource.create( project=project, name=resource_name, slug=resource_slug, @@ -108,8 +114,8 @@ def create_resource( with open(path_to_file, "r") as fh: content = fh.read() - self.api.ResourceStringsAsyncUpload.upload(content, resource=resource) - + tx_api.ResourceStringsAsyncUpload.upload(content, resource=resource) + print(f"Resource created: {resource_slug or resource_name}") else: raise Exception( f"Not project could be found wiht the slug '{project_slug}'. Please create a project first." @@ -133,9 +139,10 @@ def update_source_translation( if resource := resources.get(slug=resource_slug): with open(path_to_file, "r") as fh: content = fh.read() - self.api.ResourceStringsAsyncUpload.upload( + tx_api.ResourceStringsAsyncUpload.upload( content, resource=resource ) + print(f"Source updated for resource: {resource_slug}") return raise Exception( @@ -151,16 +158,19 @@ def get_translation( path_to_file: Path, ): """Fetch the resources matching the language given as parameter for the project""" - language = self.api.Language.get(code=language_code) + language = tx_api.Language.get(code=language_code) if project := self.get_project(project_slug=project_slug): if resources := project.fetch("resources"): if resource := resources.get(slug=resource_slug): - url = self.api.ResourceTranslationsAsyncDownload.download( + url = tx_api.ResourceTranslationsAsyncDownload.download( resource=resource, language=language ) translated_content = requests.get(url).text with open(path_to_file, "w") as fh: fh.write(translated_content) + print( + f"Translations downloaded and written to file (resource: {resource_slug})" + ) else: raise Exception( f"Unable to find any resource with this slug: '{resource_slug}'" @@ -182,7 +192,9 @@ def list_languages(self, project_slug: str) -> list[Any]: """ if self.projects: if project := self.projects.get(slug=project_slug): - return project.fetch("languages") + languages = project.fetch("languages") + print(f"Obtained these languages") + return languages raise Exception( f"Unable to find any project with this slug: '{project_slug}'" ) @@ -199,11 +211,13 @@ def create_language( ): """Create a new language resource in the remote Transifex repository""" if project := self.get_project(project_slug=project_slug): - project.add("languages", [self.api.Language.get(code=language_code)]) + project.add("languages", [tx_api.Language.get(code=language_code)]) if coordinators: project.add("coordinators", coordinators) + print(f"Created language resource for {language_code}") + @ensure_login def project_exists(self, project_slug: str) -> bool: """Check if the project exists in the remote Transifex repository""" @@ -221,27 +235,34 @@ def ping(self): Exposing this just for the sake of satisfying qgis-plugin-cli's expectations There is no need to ping the server on the current implementation, as connection is handled by the SDK """ - pass + print("'ping' is deprecated!") @ensure_login def get_translation_stats(self, project_slug: str) -> dict[str, Any]: if self.projects: if project := self.projects.get(slug=project_slug): - return self.api.ResourceLanguageStats(project=project).to_dict() + resource_stats = tx_api.ResourceLanguageStats(project=project).to_dict() + print(f"Stats acquired!: {resource_stats}") + return resource_stats raise Exception(f"Unable to find translation for this project {project_slug}") @ensure_login def push( self, project_slug: str, resource_slugs: list[str], path_to_files: list[str] ): + if len(resource_slugs) != len(path_to_files): + raise Exception( + f"Resources slugs ({len(resource_slugs)}) and Path to files ({len(path_to_files)}) must be equal in size!" + ) args = [ - tuple([pro, res, path]) - for pro, res, path in zip([project_slug], resource_slugs, path_to_files) + tuple([project_slug, res, path]) + for res, path in zip(resource_slugs, path_to_files) ] map_async( fn=self.update_source_translation, args=args, ) + print(f"Pushes some {len(resource_slugs)} files!") @ensure_login def pull( @@ -251,11 +272,13 @@ def pull( language_codes: list[str], path_to_files: list[str], ): - args = [ - tuple([pro, res, lco, path]) - for pro, res, lco, path in zip( - [project_slug], resource_slugs, language_codes, path_to_files + if len(resource_slugs) != len(path_to_files): + raise Exception( + f"Resources slugs ({len(resource_slugs)}) and Path to files ({len(path_to_files)}) must be equal in size!" ) + args = [ + tuple([project_slug, res, lco, path]) + for res, lco, path in zip(resource_slugs, language_codes, path_to_files) ] map_async( fn=self.get_translation, diff --git a/pytransifex/utils.py b/pytransifex/utils.py index 6f697aa..f20f687 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -1,8 +1,6 @@ from pathlib import Path -from typing import Any, Awaitable, Callable, Iterable, TypeAlias +from typing import Any, Callable, Iterable from asyncio import run, gather, get_running_loop -from aiofiles import open -from aiohttp import ClientSession from functools import wraps from inspect import iscoroutinefunction @@ -17,7 +15,6 @@ def capture_args(instance, *args, **kwargs): return capture_args - def map_async( *, fn: Callable | None = None, @@ -27,6 +24,8 @@ def map_async( async def closure() -> Iterable[Any]: tasks = [] if iscoroutinefunction(fn) and args: + if partials: + raise Exception("Partials don't work with coroutine functions!") tasks = [fn(*a) for a in args] else: loop = get_running_loop() @@ -34,10 +33,14 @@ async def closure() -> Iterable[Any]: tasks = [loop.run_in_executor(None, p) for p in partials] elif fn and args: tasks = [loop.run_in_executor(None, fn, *a) for a in args] + else: + raise Exception("Either partials or fn and args!") return await gather(*tasks) return run(closure()) +""" +import aiofiles, aiohttp async def write_file_async(path: Path, contents: str) -> None: async with open(path, "w") as fh: @@ -51,3 +54,4 @@ async def get_text(session: ClientSession, url: str) -> str: async with ClientSession() as session: return await gather(*[get_text(session, url) for url in urls]) +""" \ No newline at end of file diff --git a/requirements-locked.txt b/requirements-locked.txt index 490b8b1..f63d0d8 100644 --- a/requirements-locked.txt +++ b/requirements-locked.txt @@ -4,28 +4,16 @@ # # pip-compile --output-file=requirements-locked.txt requirements.txt # -aiofiles==22.1.0 - # via -r requirements.txt -aiohttp==3.8.3 - # via -r requirements.txt -aiosignal==1.2.0 - # via aiohttp alabaster==0.7.12 # via sphinx asttokens==2.0.8 # via transifex-python -async-timeout==4.0.2 - # via aiohttp -attrs==22.1.0 - # via aiohttp babel==2.10.3 # via sphinx certifi==2022.9.24 # via requests charset-normalizer==2.1.1 - # via - # aiohttp - # requests + # via requests click==8.1.3 # via # -r requirements.txt @@ -34,26 +22,16 @@ docutils==0.17.1 # via # sphinx # sphinx-rtd-theme -frozenlist==1.3.1 - # via - # aiohttp - # aiosignal future==0.18.2 # via pyseeyou idna==3.4 - # via - # requests - # yarl + # via requests imagesize==1.4.1 # via sphinx jinja2==3.1.2 # via sphinx markupsafe==2.1.1 # via jinja2 -multidict==6.0.2 - # via - # aiohttp - # yarl nose2==0.12.0 # via -r requirements.txt packaging==21.3 @@ -107,5 +85,3 @@ transifex-python==3.0.2 # via -r requirements.txt urllib3==1.26.12 # via requests -yarl==1.8.1 - # via aiohttp diff --git a/requirements.txt b/requirements.txt index 3413022..c799984 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,4 @@ sphinx_rtd_theme nose2 transifex-python click -python-dotenv -aiohttp -aiofiles \ No newline at end of file +python-dotenv \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 43dedcb..cd48790 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -32,6 +32,6 @@ def test2_map_async(self): res = map_async(partials=self.partials) assert res == self.res - def test_3_map_async(self): + def test3_map_async(self): res = map_async(fn=coroutine_fn, args=self.args) assert res == self.res \ No newline at end of file From c6e653ee891499df46bc97e502b58e7c91c50bb0 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Fri, 4 Nov 2022 17:40:22 +0100 Subject: [PATCH 11/34] Concurrency simplification and enhancement. --- pytransifex/api_new.py | 8 +++---- pytransifex/cli.py | 1 + pytransifex/utils.py | 48 +++++++++--------------------------------- tests/test_cli.py | 5 +++-- tests/test_utils.py | 21 ++++++------------ 5 files changed, 24 insertions(+), 59 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 6f4a520..b12a14c 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,11 +1,11 @@ -import requests - -from typing import Any from pathlib import Path +from typing import Any + +import requests from transifex.api import transifex_api as tx_api -from transifex.api.jsonapi.resources import Resource from transifex.api.jsonapi import JsonApiException from transifex.api.jsonapi.exceptions import DoesNotExist +from transifex.api.jsonapi.resources import Resource from pytransifex.config import Config from pytransifex.interfaces import Tx diff --git a/pytransifex/cli.py b/pytransifex/cli.py index 002643b..04ac8f6 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -1,4 +1,5 @@ from typing import Any + import click from pytransifex.api_new import Transifex diff --git a/pytransifex/utils.py b/pytransifex/utils.py index f20f687..b3c48c3 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -1,8 +1,6 @@ -from pathlib import Path -from typing import Any, Callable, Iterable -from asyncio import run, gather, get_running_loop +from concurrent.futures import ThreadPoolExecutor, as_completed from functools import wraps -from inspect import iscoroutinefunction +from typing import Any, Callable, Iterable def ensure_login(f): @@ -20,38 +18,12 @@ def map_async( fn: Callable | None = None, args: Iterable[Any] | None = None, partials: Iterable[Any] | None = None, -) -> Iterable[Any]: - async def closure() -> Iterable[Any]: - tasks = [] - if iscoroutinefunction(fn) and args: - if partials: - raise Exception("Partials don't work with coroutine functions!") - tasks = [fn(*a) for a in args] +) -> list[Any]: + with ThreadPoolExecutor() as pool: + if partials: + futures = [pool.submit(p) for p in partials] + elif fn and args: + futures = [pool.submit(fn, *a) for a in args] else: - loop = get_running_loop() - if partials: - tasks = [loop.run_in_executor(None, p) for p in partials] - elif fn and args: - tasks = [loop.run_in_executor(None, fn, *a) for a in args] - else: - raise Exception("Either partials or fn and args!") - return await gather(*tasks) - - return run(closure()) - -""" -import aiofiles, aiohttp - -async def write_file_async(path: Path, contents: str) -> None: - async with open(path, "w") as fh: - await fh.write(contents) - - -async def get_async_from_url(urls: list[str]) -> list[str]: - async def get_text(session: ClientSession, url: str) -> str: - response = await session.get(url) - return await response.text() - - async with ClientSession() as session: - return await gather(*[get_text(session, url) for url in urls]) -""" \ No newline at end of file + raise Exception("Either partials or fn and args!") + return [f.result() for f in as_completed(futures)] diff --git a/tests/test_cli.py b/tests/test_cli.py index ebe1dc2..c083f4c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,10 @@ import unittest + from click.testing import CliRunner -from pytransifex.config import Config -from pytransifex.cli import cli from pytransifex.api_old import Transifex +from pytransifex.cli import cli +from pytransifex.config import Config class TestCli(unittest.TestCase): diff --git a/tests/test_utils.py b/tests/test_utils.py index cd48790..ece505d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,16 +1,11 @@ import unittest -from pytransifex.utils import map_async from functools import partial -from asyncio import sleep as asleep from time import sleep as tsleep - -async def coroutine_fn(a: int, b: int) -> int: - await asleep(a) - return a + b +from pytransifex.utils import map_async -def non_coroutine_fn(a: int, b: int) -> int: +def fn(a: int, b: int) -> int: tsleep(a) return a + b @@ -18,20 +13,16 @@ def non_coroutine_fn(a: int, b: int) -> int: class TestUtils(unittest.TestCase): @classmethod def setUpClass(cls): - it = [1,2,3] + it = [1, 2, 3] cls.it = it cls.args = [tuple([a, b]) for a, b in zip(it, it)] - cls.partials = [partial(non_coroutine_fn, a, b) for a, b in zip(it, it)] + cls.partials = [partial(fn, a, b) for a, b in zip(it, it)] cls.res = [2, 4, 6] def test1_map_async(self): - res = map_async(fn=coroutine_fn, args=self.args) + res = map_async(partials=self.partials) assert res == self.res def test2_map_async(self): - res = map_async(partials=self.partials) + res = map_async(fn=fn, args=self.args) assert res == self.res - - def test3_map_async(self): - res = map_async(fn=coroutine_fn, args=self.args) - assert res == self.res \ No newline at end of file From 8fc06fac124ff25d175f3140659092aa508ac84f Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Mon, 7 Nov 2022 12:48:16 +0100 Subject: [PATCH 12/34] CLI interface, CLI tests --- pytransifex/api_new.py | 64 +++++++++++--------- pytransifex/cli.py | 132 +++++++++++++++++++++++++++++++++++------ pytransifex/config.py | 12 ++++ pytransifex/utils.py | 2 +- tests/test_api_new.py | 2 +- tests/test_cli.py | 54 +++++++++++++---- tests/test_utils.py | 6 +- 7 files changed, 211 insertions(+), 61 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index b12a14c..6b9b9f9 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -9,7 +9,7 @@ from pytransifex.config import Config from pytransifex.interfaces import Tx -from pytransifex.utils import ensure_login, map_async +from pytransifex.utils import concurrently, ensure_login class Client(Tx): @@ -155,22 +155,29 @@ def get_translation( project_slug: str, resource_slug, language_code: str, - path_to_file: Path, + output_dir: Path, ): """Fetch the resources matching the language given as parameter for the project""" language = tx_api.Language.get(code=language_code) + path_to_file = output_dir.joinpath(f"{resource_slug}.{language_code}") + if project := self.get_project(project_slug=project_slug): + if resources := project.fetch("resources"): + if resource := resources.get(slug=resource_slug): url = tx_api.ResourceTranslationsAsyncDownload.download( resource=resource, language=language ) translated_content = requests.get(url).text + with open(path_to_file, "w") as fh: fh.write(translated_content) + print( f"Translations downloaded and written to file (resource: {resource_slug})" ) + else: raise Exception( f"Unable to find any resource with this slug: '{resource_slug}'" @@ -238,53 +245,56 @@ def ping(self): print("'ping' is deprecated!") @ensure_login - def get_translation_stats(self, project_slug: str) -> dict[str, Any]: + def get_project_stats(self, project_slug: str) -> dict[str, Any]: if self.projects: if project := self.projects.get(slug=project_slug): - resource_stats = tx_api.ResourceLanguageStats(project=project).to_dict() - print(f"Stats acquired!: {resource_stats}") - return resource_stats + print(str(project)) + if resource_stats := tx_api.ResourceLanguageStats.get( + "o:test_pytransifex:p:test_project_pytransifex" + ): + print(f"Stats acquired!: {resource_stats}") + return resource_stats.to_dict() raise Exception(f"Unable to find translation for this project {project_slug}") @ensure_login - def push( - self, project_slug: str, resource_slugs: list[str], path_to_files: list[str] + def pull( + self, + project_slug: str, + resource_slugs: list[str], + language_codes: list[str], + output_dir: Path, ): - if len(resource_slugs) != len(path_to_files): - raise Exception( - f"Resources slugs ({len(resource_slugs)}) and Path to files ({len(path_to_files)}) must be equal in size!" - ) args = [ - tuple([project_slug, res, path]) - for res, path in zip(resource_slugs, path_to_files) + tuple([project_slug, slug, lcode, output_dir]) + for slug, lcode in zip(resource_slugs, language_codes) ] - map_async( - fn=self.update_source_translation, + + concurrently( + fn=self.get_translation, args=args, ) - print(f"Pushes some {len(resource_slugs)} files!") @ensure_login - def pull( - self, - project_slug: str, - resource_slugs: list[str], - language_codes: list[str], - path_to_files: list[str], + def push( + self, project_slug: str, resource_slugs: list[str], path_to_files: list[Path] ): if len(resource_slugs) != len(path_to_files): raise Exception( f"Resources slugs ({len(resource_slugs)}) and Path to files ({len(path_to_files)}) must be equal in size!" ) + args = [ - tuple([project_slug, res, lco, path]) - for res, lco, path in zip(resource_slugs, language_codes, path_to_files) + tuple([project_slug, res, path]) + for res, path in zip(resource_slugs, path_to_files) ] - map_async( - fn=self.get_translation, + + concurrently( + fn=self.update_source_translation, args=args, ) + print(f"Pushes some {len(resource_slugs)} files!") + class Transifex: """ diff --git a/pytransifex/cli.py b/pytransifex/cli.py index 04ac8f6..d8e6958 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -1,37 +1,133 @@ -from typing import Any +from dataclasses import asdict +from os import mkdir, rmdir +from pathlib import Path import click from pytransifex.api_new import Transifex +from pytransifex.config import CliSettings client = Transifex(defer_login=True) +cli_settings: CliSettings | None = None -def format_args(args: dict[str, Any]) -> dict[str, Any]: - return {k.replace("-", "_"): v for k, v in args.items()} +@click.group(chain=True) +@click.pass_context +def cli(ctx): + ctx.obj = {} -@click.group() -def cli(): - pass +@click.option("-v", "--verbose", is_flag=True, default=False) +@click.option("-out", "--output-dir", is_flag=False) +@click.option("-in", "--input-dir", is_flag=False) +@click.option("-org", "--organization-slug", is_flag=False) +@click.option("-p", "--project-slug", is_flag=False) +@cli.command( + "init", help="Initialize the CLI with the appropriate configuration values." +) +@click.pass_context +def init(ctx, **opts): + mandatory = ["organization_slug", "project_slug"] + if missing := [k for k in mandatory if not k in opts]: + raise Exception( + f"These keys are not set or do not have a well-defined value: {', '.join(missing)}" + ) + if empty := [k for k, v in opts.items() if k in mandatory and not v]: + raise Exception(f"These keys have an empty value: {', '.join(empty)}") + organization_slug = opts["organization_slug"] + project_slug = opts["project_slug"] -@click.option("--verbose", is_flag=True, default=False) -@click.argument("file_name", required=True) + input_directory = opts.get("input_dir") + output_directory = opts.get("input_dir") + input_directory = Path(input_directory) if input_directory else Path.cwd() + output_directory = ( + Path(output_directory) + if output_directory + else Path.cwd().joinpath("downloaded_files") + ) + + has_to_create_dir = not Path.exists(output_directory) + reply = "" + + try: + click.echo(f"Initializing...") + + if has_to_create_dir: + mkdir(output_directory) + + settings = asdict(CliSettings( + organization_slug, project_slug, input_directory, output_directory + )) + + for k, v in settings.items(): + ctx.obj[k] = v + + reply += f"Initialized project with the following settings: {project_slug}. " + + except Exception as error: + reply += f"Failed to initialize the CLI, this error occurred: {error} " + + if has_to_create_dir: + rmdir(output_directory) + + reply += f"Removed {output_directory}. " + + finally: + click.echo(reply) + + +@click.argument("input-directory", required=False) @cli.command("push", help="Push translation strings") -def push(file_name: str, verbose: bool): - click.echo(f"Pushing: {file_name}") - client.push() - if verbose: - click.echo("Was it verbose enough?") +@click.pass_context +def push(ctx, input_directory: str | None): + print(f"CONTEXT from PUSH: {ctx.obj}") + if input_directory: + ctx.obj["input_directory"] = Path(input_directory) + + resource_slugs = [""] + path_to_files = list( + Path.iterdir( + ctx.obj["input_directory"] + if "input_directory" in ctx.obj + else Path.cwd() + ) + ) + reply = "" + try: + click.echo( + f"Pushing {resource_slugs} from {path_to_files} to Transifex under project {ctx.obj['project_slug']}..." + ) + # client.push(project_slug=ctx.project_slug, resource_slugs=resource_slugs, path_to_files=path_to_files) + except Exception as error: + reply += f"Failed because of this error: {error}" + finally: + click.echo(reply) -@click.option("--only-lang", "-l", default="all") -@click.argument("dir_name", required=True) + +@click.option("-l", "--only-lang", default="all") +@click.argument("output-directory", required=False) @cli.command("pull", help="Pull translation strings") -def pull(dir_name: str, only_lang: str): - client.pull() - click.echo(f"Pulling: {dir_name}, {only_lang}") +@click.pass_context +def pull(ctx, output_directory: str | None, only_lang: str | None): + print(f"CONTEXT from PULL: {ctx.obj}") + if output_directory: + ctx.obj["output_dir"] = output_directory + + resource_slugs = [] + reply = "" + language_codes = only_lang.split(",") if only_lang else [] + + try: + click.echo( + f"Pulling translation strings from project {ctx.obj['project_slug']} to {output_directory}..." + ) + # client.pull(project_slug=ctx.project_slug, resource_slugs=resource_slugs, language_codes=language_codes, output_dir=ctx.output_dir) + except Exception as error: + reply += f"Failed because of this error: {error}" + finally: + click.echo(reply) if __name__ == "__main__": diff --git a/pytransifex/config.py b/pytransifex/config.py index 1e62d32..92fbbcb 100644 --- a/pytransifex/config.py +++ b/pytransifex/config.py @@ -1,5 +1,8 @@ +from dataclasses import dataclass from os import environ +from pathlib import Path from typing import NamedTuple + from dotenv import load_dotenv @@ -8,6 +11,7 @@ class Config(NamedTuple): organization_name: str i18n_type: str host_name = "https://rest.api.transifex.com" + project_slug: str | None = None @classmethod def from_env(cls) -> "Config": @@ -23,3 +27,11 @@ def from_env(cls) -> "Config": ) return cls(token, organization, i18n_type) # type: ignore + + +@dataclass +class CliSettings: + organization_slug: str + project_slug: str | None + input_directory: Path | None + output_directory: Path | None diff --git a/pytransifex/utils.py b/pytransifex/utils.py index b3c48c3..26e894c 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -13,7 +13,7 @@ def capture_args(instance, *args, **kwargs): return capture_args -def map_async( +def concurrently( *, fn: Callable | None = None, args: Iterable[Any] | None = None, diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 7b0f49f..1210864 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -79,7 +79,7 @@ def test10_ping(self): """ def test11_stats(self): - stats = self.tx.get_translation_stats(project_slug=self.project_slug) + stats = self.tx.get_project_stats(project_slug=self.project_slug) print(dir(stats)) print(str(stats)) assert stats diff --git a/tests/test_cli.py b/tests/test_cli.py index c083f4c..5f05bd0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,22 +1,54 @@ import unittest +from pathlib import Path from click.testing import CliRunner -from pytransifex.api_old import Transifex +from pytransifex.api_new import Transifex from pytransifex.cli import cli -from pytransifex.config import Config class TestCli(unittest.TestCase): - def test_cli(self): - config = Config("token", "organization", "po") - _ = Transifex(config) - runner = CliRunner() - result = runner.invoke(cli, ["pull", "somedir", "-l", "fr"]) - - if result.exit_code != 0: - print(result.output) - assert result.exit_code == 0 + @classmethod + def setUpClass(cls): + """ + cls.tx = Transifex(defer_login=True) + cls.project_slug = "test_project_pytransifex" + cls.project_name = "Test Project PyTransifex" + cls.resource_slug = "test_resource_fr" + cls.resource_name = "Test Resource FR" + cls.path_to_file = Path(Path.cwd()).joinpath("tests", "test_resource_fr.po") + + if project := cls.tx.get_project(project_slug=cls.project_slug): + print("Found old project, removing.") + project.delete() + + print("Creating a brand new project") + cls.tx.create_project( + project_name=cls.project_name, project_slug=cls.project_slug, private=True + ) + """ + cls.runner = CliRunner() + + def test1_init(self): + result = self.runner.invoke( + cli, + ["init", "-out", "OUTPUT", "-org", "ORGA", "-in", "INPUT", "-p", "PROJECT"], + ) + passed = result.exit_code == 0 + print(result.output if passed else result) + assert passed + + def test2_push(self): + result = self.runner.invoke(cli, ["push"]) + passed = result.exit_code == 0 + print(result.output if passed else result) + assert passed + + def test3_pull(self): + result = self.runner.invoke(cli, ["pull", "-l", "de,fr,it,en"]) + passed = result.exit_code == 0 + print(result.output if passed else result) + assert passed if __name__ == "__main__": diff --git a/tests/test_utils.py b/tests/test_utils.py index ece505d..b5d4b35 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,7 +2,7 @@ from functools import partial from time import sleep as tsleep -from pytransifex.utils import map_async +from pytransifex.utils import concurrently def fn(a: int, b: int) -> int: @@ -20,9 +20,9 @@ def setUpClass(cls): cls.res = [2, 4, 6] def test1_map_async(self): - res = map_async(partials=self.partials) + res = concurrently(partials=self.partials) assert res == self.res def test2_map_async(self): - res = map_async(fn=fn, args=self.args) + res = concurrently(fn=fn, args=self.args) assert res == self.res From 025d6431a3460ac47397aee99a12c669742ad582 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 8 Nov 2022 12:34:46 +0100 Subject: [PATCH 13/34] CLI passing initial tests. --- pytransifex/api_new.py | 17 +++-- pytransifex/api_old.py | 4 +- pytransifex/cli.py | 134 +++++++++++++++++++--------------------- pytransifex/config.py | 69 +++++++++++++++++++-- requirements-locked.txt | 2 + requirements.txt | 3 +- tests/test_api_new.py | 27 +++++--- tests/test_api_old.py | 4 +- tests/test_cli.py | 14 +++-- 9 files changed, 171 insertions(+), 103 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 6b9b9f9..d64b55e 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -7,7 +7,7 @@ from transifex.api.jsonapi.exceptions import DoesNotExist from transifex.api.jsonapi.resources import Resource -from pytransifex.config import Config +from pytransifex.config import ApiConfig from pytransifex.interfaces import Tx from pytransifex.utils import concurrently, ensure_login @@ -18,7 +18,7 @@ class Client(Tx): By default instances are created and logged in 'lazyly' -- when creation or login cannot be deferred any longer. """ - def __init__(self, config: Config, defer_login: bool = False): + def __init__(self, config: ApiConfig, defer_login: bool = False): """Extract config values, consumes API token against SDK client""" self.api_token = config.api_token self.host = config.host_name @@ -153,11 +153,11 @@ def update_source_translation( def get_translation( self, project_slug: str, - resource_slug, + resource_slug: str, language_code: str, output_dir: Path, ): - """Fetch the resources matching the language given as parameter for the project""" + """Fetch the translation resource matching the given language""" language = tx_api.Language.get(code=language_code) path_to_file = output_dir.joinpath(f"{resource_slug}.{language_code}") @@ -248,12 +248,9 @@ def ping(self): def get_project_stats(self, project_slug: str) -> dict[str, Any]: if self.projects: if project := self.projects.get(slug=project_slug): - print(str(project)) - if resource_stats := tx_api.ResourceLanguageStats.get( - "o:test_pytransifex:p:test_project_pytransifex" - ): - print(f"Stats acquired!: {resource_stats}") + if resource_stats := tx_api.ResourceLanguageStats(project=project): return resource_stats.to_dict() + raise Exception(f"Unable to find translation for this project {project_slug}") @ensure_login @@ -306,5 +303,5 @@ class Transifex: def __new__(cls, defer_login: bool = False): if not cls.client: - cls.client = Client(Config.from_env(), defer_login) + cls.client = Client(ApiConfig.from_env(), defer_login) return cls.client diff --git a/pytransifex/api_old.py b/pytransifex/api_old.py index b31e62b..cf62525 100755 --- a/pytransifex/api_old.py +++ b/pytransifex/api_old.py @@ -5,12 +5,12 @@ from typing import Any from types import FunctionType -from pytransifex.config import Config +from pytransifex.config import ApiConfig from pytransifex.exceptions import PyTransifexException class Transifex: - def __init__(self, config: Config): + def __init__(self, config: ApiConfig): """ Initializes Transifex diff --git a/pytransifex/cli.py b/pytransifex/cli.py index d8e6958..42aab7d 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -1,4 +1,3 @@ -from dataclasses import asdict from os import mkdir, rmdir from pathlib import Path @@ -11,123 +10,120 @@ cli_settings: CliSettings | None = None -@click.group(chain=True) -@click.pass_context -def cli(ctx): - ctx.obj = {} +def extract_files(input_dir: Path) -> tuple[list[Path], list[str], str]: + files = list(Path.iterdir(input_dir)) + slugs = [str(f).split(".")[0] for f in files] + files_status_report = "\n".join( + (f"{slug} => {file}" for slug, file in zip(slugs, files)) + ) + return (files, slugs, files_status_report) + + +@click.group +def cli(): + pass @click.option("-v", "--verbose", is_flag=True, default=False) -@click.option("-out", "--output-dir", is_flag=False) -@click.option("-in", "--input-dir", is_flag=False) +@click.option("-out", "--output-directory", is_flag=False) +@click.option("-in", "--input-directory", is_flag=False) @click.option("-org", "--organization-slug", is_flag=False) @click.option("-p", "--project-slug", is_flag=False) @cli.command( "init", help="Initialize the CLI with the appropriate configuration values." ) -@click.pass_context -def init(ctx, **opts): - mandatory = ["organization_slug", "project_slug"] - if missing := [k for k in mandatory if not k in opts]: - raise Exception( - f"These keys are not set or do not have a well-defined value: {', '.join(missing)}" - ) - if empty := [k for k, v in opts.items() if k in mandatory and not v]: - raise Exception(f"These keys have an empty value: {', '.join(empty)}") - - organization_slug = opts["organization_slug"] - project_slug = opts["project_slug"] - - input_directory = opts.get("input_dir") - output_directory = opts.get("input_dir") - input_directory = Path(input_directory) if input_directory else Path.cwd() - output_directory = ( - Path(output_directory) - if output_directory - else Path.cwd().joinpath("downloaded_files") - ) - - has_to_create_dir = not Path.exists(output_directory) +def init(**opts): reply = "" + settings = CliSettings.extract_settings(**opts) + has_to_create_dir = not Path.exists(settings.output_directory) try: click.echo(f"Initializing...") - if has_to_create_dir: - mkdir(output_directory) + if has_to_create_dir: + mkdir(settings.output_directory) + + reply += f"Initialized project with the following settings: {settings.project_slug} and saved file to {settings.config_file} " - settings = asdict(CliSettings( - organization_slug, project_slug, input_directory, output_directory - )) - - for k, v in settings.items(): - ctx.obj[k] = v - - reply += f"Initialized project with the following settings: {project_slug}. " + if not settings.input_directory: + reply += f"WARNING: You will need to declare an input directory if you plan on using 'pytx push', as in 'pytx push --input-directory '." except Exception as error: reply += f"Failed to initialize the CLI, this error occurred: {error} " if has_to_create_dir: - rmdir(output_directory) + rmdir(settings.output_directory) - reply += f"Removed {output_directory}. " + reply += f"Removed {settings.output_directory}. " finally: click.echo(reply) + settings.to_disk() -@click.argument("input-directory", required=False) +@click.option("-in", "--input-directory", is_flag=False) @cli.command("push", help="Push translation strings") -@click.pass_context -def push(ctx, input_directory: str | None): - print(f"CONTEXT from PUSH: {ctx.obj}") - if input_directory: - ctx.obj["input_directory"] = Path(input_directory) - - resource_slugs = [""] - path_to_files = list( - Path.iterdir( - ctx.obj["input_directory"] - if "input_directory" in ctx.obj - else Path.cwd() - ) - ) +def push(input_directory: str | None): reply = "" + settings = CliSettings.from_disk() + input_dir = ( + Path(input_directory) + if input_directory + else getattr(settings, "input_directory", None) + ) + + if not input_dir: + raise Exception( + "To use this 'push', you need to initialize the project with a valid path to the directory containing the files to push; alternatively, you can call this commend with 'pytx push --input-directory '." + ) try: + files, slugs, files_status_report = extract_files(input_dir) click.echo( - f"Pushing {resource_slugs} from {path_to_files} to Transifex under project {ctx.obj['project_slug']}..." + f"Pushing {files_status_report} to Transifex under project {settings.project_slug}..." + ) + client.push( + project_slug=settings.project_slug, + resource_slugs=slugs, + path_to_files=files, ) - # client.push(project_slug=ctx.project_slug, resource_slugs=resource_slugs, path_to_files=path_to_files) except Exception as error: reply += f"Failed because of this error: {error}" finally: click.echo(reply) + settings.to_disk() @click.option("-l", "--only-lang", default="all") -@click.argument("output-directory", required=False) +@click.option("-out", "--output-directory", is_flag=False) @cli.command("pull", help="Pull translation strings") -@click.pass_context -def pull(ctx, output_directory: str | None, only_lang: str | None): - print(f"CONTEXT from PULL: {ctx.obj}") - if output_directory: - ctx.obj["output_dir"] = output_directory - - resource_slugs = [] +def pull(output_directory: str | Path | None, only_lang: str | None): reply = "" + settings = CliSettings.from_disk() + resource_slugs = [] language_codes = only_lang.split(",") if only_lang else [] + if output_directory: + output_directory = Path(output_directory) + settings.output_directory = output_directory + else: + output_directory = settings.output_directory + try: click.echo( - f"Pulling translation strings from project {ctx.obj['project_slug']} to {output_directory}..." + f"Pulling translation strings ({language_codes}) from project {settings.project_slug} to {str(output_directory)}..." + ) + client.pull( + project_slug=settings.project_slug, + resource_slugs=resource_slugs, + language_codes=language_codes, + output_dir=output_directory, ) - # client.pull(project_slug=ctx.project_slug, resource_slugs=resource_slugs, language_codes=language_codes, output_dir=ctx.output_dir) except Exception as error: reply += f"Failed because of this error: {error}" finally: click.echo(reply) + settings.to_disk() if __name__ == "__main__": diff --git a/pytransifex/config.py b/pytransifex/config.py index 92fbbcb..8bd7454 100644 --- a/pytransifex/config.py +++ b/pytransifex/config.py @@ -1,12 +1,13 @@ -from dataclasses import dataclass +from dataclasses import asdict, dataclass from os import environ from pathlib import Path -from typing import NamedTuple +from typing import Any, NamedTuple +import toml from dotenv import load_dotenv -class Config(NamedTuple): +class ApiConfig(NamedTuple): api_token: str organization_name: str i18n_type: str @@ -14,7 +15,7 @@ class Config(NamedTuple): project_slug: str | None = None @classmethod - def from_env(cls) -> "Config": + def from_env(cls) -> "ApiConfig": load_dotenv() token = environ.get("TX_TOKEN") @@ -29,9 +30,65 @@ def from_env(cls) -> "Config": return cls(token, organization, i18n_type) # type: ignore +path_keys = ["input_directory", "output_directory", "config_file"] +mandatory = ["organization_slug", "project_slug"] +defaults = { + "output_directory": Path.cwd().joinpath("downloaded_files"), + "config_file": Path.cwd().joinpath(".pytx_config.yaml"), +} + + @dataclass class CliSettings: organization_slug: str - project_slug: str | None + project_slug: str input_directory: Path | None - output_directory: Path | None + output_directory: Path = defaults["output_directory"] + config_file: Path = defaults["config_file"] + + @classmethod + def extract_settings(cls, **user_data) -> "CliSettings": + + if missing := [k for k in mandatory if not k in user_data]: + raise Exception( + f"These keys are not set or do not have a well-defined value: {', '.join(missing)}" + ) + if empty := [k for k, v in user_data.items() if k in mandatory and not v]: + raise Exception(f"These keys have an empty value: {', '.join(empty)}") + + organization_slug = user_data["organization_slug"] + project_slug = user_data["project_slug"] + input_directory = user_data.get("input_directory") + config_file = CliSettings.get_or_default("config_file", user_data) + output_directory = CliSettings.get_or_default("output_directory", user_data) + + return cls( + organization_slug, + project_slug, + input_directory, + output_directory, + config_file, + ) + + @classmethod + def from_disk(cls) -> "CliSettings": + d = toml.load(cls.config_file) + return cls.extract_settings(**d) + + def to_disk(self): + with open(self.config_file, "w") as fh: + toml.dump(self.serialize(), fh) + + def serialize(self) -> dict[str, Any]: + return {k: (str(v) if k in path_keys else v) for k, v in asdict(self).items() if v is not None} + + @staticmethod + def deserialize(d: dict[str, Any]) -> dict[str, Any]: + return {k: (Path(v) if k in path_keys else v) for k, v in d.items()} + + @staticmethod + def get_or_default(k: str, obj: dict[str, Any]) -> Any: + truthy_obj = {k: v for k, v in obj.items() if v} + if k in truthy_obj: + return truthy_obj[k] + return defaults[k] diff --git a/requirements-locked.txt b/requirements-locked.txt index f63d0d8..be34f79 100644 --- a/requirements-locked.txt +++ b/requirements-locked.txt @@ -79,6 +79,8 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx +toml==0.10.2 + # via -r requirements.txt toolz==0.12.0 # via pyseeyou transifex-python==3.0.2 diff --git a/requirements.txt b/requirements.txt index c799984..70e5dc5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ sphinx_rtd_theme nose2 transifex-python click -python-dotenv \ No newline at end of file +python-dotenv +toml \ No newline at end of file diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 1210864..577d3f8 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -27,7 +27,6 @@ def setUpClass(cls): def test1_new_api_satisfies_abc(self): assert isinstance(self.tx, Tx) - """ def test2_create_project(self): # Done in setUpClass pass @@ -58,7 +57,7 @@ def test6_get_translation(self): project_slug=self.project_slug, resource_slug=self.resource_slug, language_code="fr_CH", - path_to_file=self.path_to_file, + output_dir=self.path_to_file, ) assert translation @@ -76,22 +75,32 @@ def test9_project_exists(self): def test10_ping(self): self.tx.ping() assert True - """ def test11_stats(self): stats = self.tx.get_project_stats(project_slug=self.project_slug) - print(dir(stats)) print(str(stats)) assert stats - """ def test12_stats(self): - self.tx.get_translation_stats(project_slug=self.project_slug) - + self.tx.get_project_stats(project_slug=self.project_slug) + def test13_stats(self): - self.tx.get_translation_stats(project_slug=self.project_slug) - """ + self.tx.get_project_stats(project_slug=self.project_slug) if __name__ == "__main__": unittest.main() + +""" +# Don't remove this! +curl -g \ + --request GET --url "https://rest.api.transifex.com/resource_language_stats?filter[project]=o%3Aopengisch%3Ap%3Aqfield-documentation" \ + --header 'accept: application/vnd.api+json' \ + --header 'authorization: Bearer TOKEN' + +curl -g \ + --request GET \ + --url "https://rest.api.transifex.com/resource_language_stats?filter[project]=o%3Atest_pytransifex%3Ap%3Atest_project_pytransifex" \ + --header 'accept: application/vnd.api+json' \ + --header 'authorization: Bearer TOKEN' +""" diff --git a/tests/test_api_old.py b/tests/test_api_old.py index 93cc502..282cc66 100644 --- a/tests/test_api_old.py +++ b/tests/test_api_old.py @@ -3,7 +3,7 @@ import unittest import os -from pytransifex.config import Config +from pytransifex.config import ApiConfig from pytransifex.exceptions import PyTransifexException from pytransifex.api_old import Transifex @@ -13,7 +13,7 @@ def setUp(self): token = os.getenv("TX_TOKEN") assert token is not None self.t = Transifex( - Config(organization="pytransifex", api_token=token, i18n_type="PO") + ApiConfig(organization="pytransifex", api_token=token, i18n_type="PO") ) self.project_slug = "pytransifex-test-project" self.project_name = "PyTransifex Test project" diff --git a/tests/test_cli.py b/tests/test_cli.py index 5f05bd0..9824391 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,10 +1,11 @@ import unittest from pathlib import Path - +from os import remove from click.testing import CliRunner from pytransifex.api_new import Transifex from pytransifex.cli import cli +from pytransifex.config import CliSettings, defaults class TestCli(unittest.TestCase): @@ -29,18 +30,23 @@ def setUpClass(cls): """ cls.runner = CliRunner() + @classmethod + def tearDownClass(cls): + if Path.exists(defaults["config_file"]): + remove(defaults["config_file"]) + def test1_init(self): result = self.runner.invoke( cli, - ["init", "-out", "OUTPUT", "-org", "ORGA", "-in", "INPUT", "-p", "PROJECT"], + ["init", "-org", "ORGA", "-p", "PROJECT"], ) passed = result.exit_code == 0 print(result.output if passed else result) assert passed def test2_push(self): - result = self.runner.invoke(cli, ["push"]) - passed = result.exit_code == 0 + result = self.runner.invoke(cli, ["push", "-in", "SOME_DIRECTORY"]) + passed = result.exit_code == 0 and "No such file" in result.output print(result.output if passed else result) assert passed From ca97edf6da60d0def775f91b6c1d937afb3d8704 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 22 Nov 2022 09:39:51 +0100 Subject: [PATCH 14/34] Fixed last failing test. --- pytransifex/api_new.py | 8 ++++++-- tests/test_api_new.py | 21 +++++++++++---------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index d64b55e..fb78346 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,3 +1,4 @@ +from os import mkdir from pathlib import Path from typing import Any @@ -159,7 +160,7 @@ def get_translation( ): """Fetch the translation resource matching the given language""" language = tx_api.Language.get(code=language_code) - path_to_file = output_dir.joinpath(f"{resource_slug}.{language_code}") + file_name = Path.joinpath(output_dir, resource_slug) if project := self.get_project(project_slug=project_slug): @@ -171,7 +172,10 @@ def get_translation( ) translated_content = requests.get(url).text - with open(path_to_file, "w") as fh: + if not Path.exists(output_dir): + mkdir(output_dir) + + with open(file_name, "w") as fh: fh.write(translated_content) print( diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 577d3f8..57813e1 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -14,6 +14,7 @@ def setUpClass(cls): cls.resource_slug = "test_resource_fr" cls.resource_name = "Test Resource FR" cls.path_to_file = Path(Path.cwd()).joinpath("tests", "test_resource_fr.po") + cls.output_dir = Path(Path.cwd()).joinpath("tests", "output") if project := cls.tx.get_project(project_slug=cls.project_slug): print("Found old project, removing.") @@ -52,21 +53,21 @@ def test5_update_source_translation(self): ) assert True - def test6_get_translation(self): - translation = self.tx.get_translation( - project_slug=self.project_slug, - resource_slug=self.resource_slug, - language_code="fr_CH", - output_dir=self.path_to_file, - ) - assert translation + def test6_create_language(self): + self.tx.create_language(self.project_slug, "fr_CH") def test7_list_languages(self): languages = self.tx.list_languages(project_slug=self.project_slug) assert languages is not None - def test8_create_language(self): - self.tx.create_language(self.project_slug, "fr_CH") + def test8_get_translation(self): + self.tx.get_translation( + project_slug=self.project_slug, + resource_slug=self.resource_slug, + language_code="fr_CH", + output_dir=self.output_dir, + ) + assert Path.exists(Path.joinpath(self.output_dir, self.resource_slug)) def test9_project_exists(self): verdict = self.tx.project_exists(project_slug=self.project_slug) From dbbf1cf34704cfb08fc8d0c8156e84ab97926107 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 22 Nov 2022 09:50:23 +0100 Subject: [PATCH 15/34] Added cleanup for tests. --- .env.example | 2 ++ .pytx_config.yml | 4 ++++ pytransifex/api_new.py | 12 ++++++------ pytransifex/cli.py | 13 +++++++++---- pytransifex/config.py | 4 ++-- tests/{ => input}/test_resource_fr.po | 0 tests/test_api_new.py | 10 ++++++++-- tests/test_cli.py | 23 +++++++++++------------ 8 files changed, 42 insertions(+), 26 deletions(-) create mode 100644 .env.example create mode 100644 .pytx_config.yml rename tests/{ => input}/test_resource_fr.po (100%) diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cfda38d --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +TX_TOKEN=0123456789 +ORGANIZATION=test_pytransifex \ No newline at end of file diff --git a/.pytx_config.yml b/.pytx_config.yml new file mode 100644 index 0000000..cf86124 --- /dev/null +++ b/.pytx_config.yml @@ -0,0 +1,4 @@ +organization_slug = "test_pytransifex" +project_slug = "test_project_pytransifex" +output_directory = "/home/hades/opengis/pytransifex/output" +config_file = "/home/hades/opengis/pytransifex/.pytx_config.yml" diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index fb78346..c4e909c 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -265,11 +265,11 @@ def pull( language_codes: list[str], output_dir: Path, ): - args = [ - tuple([project_slug, slug, lcode, output_dir]) - for slug, lcode in zip(resource_slugs, language_codes) - ] - + args = [] + for l_code in language_codes: + for slug in resource_slugs: + args.append(tuple([project_slug, slug, l_code, output_dir])) + print("ARGS", args) concurrently( fn=self.get_translation, args=args, @@ -288,7 +288,7 @@ def push( tuple([project_slug, res, path]) for res, path in zip(resource_slugs, path_to_files) ] - + concurrently( fn=self.update_source_translation, args=args, diff --git a/pytransifex/cli.py b/pytransifex/cli.py index 42aab7d..fd616e2 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -7,12 +7,17 @@ from pytransifex.config import CliSettings client = Transifex(defer_login=True) -cli_settings: CliSettings | None = None + + +def path_to_slug(file_paths: list[Path]) -> list[str]: + keep_last = (str(f).split("/")[-1] for f in file_paths) + remove_dot = (s.split(".")[0] for s in keep_last) + return list(remove_dot) def extract_files(input_dir: Path) -> tuple[list[Path], list[str], str]: files = list(Path.iterdir(input_dir)) - slugs = [str(f).split(".")[0] for f in files] + slugs = path_to_slug(files) files_status_report = "\n".join( (f"{slug} => {file}" for slug, file in zip(slugs, files)) ) @@ -67,7 +72,7 @@ def push(input_directory: str | None): reply = "" settings = CliSettings.from_disk() input_dir = ( - Path(input_directory) + Path.cwd().joinpath(input_directory) if input_directory else getattr(settings, "input_directory", None) ) @@ -100,7 +105,6 @@ def push(input_directory: str | None): def pull(output_directory: str | Path | None, only_lang: str | None): reply = "" settings = CliSettings.from_disk() - resource_slugs = [] language_codes = only_lang.split(",") if only_lang else [] if output_directory: @@ -109,6 +113,7 @@ def pull(output_directory: str | Path | None, only_lang: str | None): else: output_directory = settings.output_directory + resource_slugs = [] try: click.echo( f"Pulling translation strings ({language_codes}) from project {settings.project_slug} to {str(output_directory)}..." diff --git a/pytransifex/config.py b/pytransifex/config.py index 8bd7454..004f2ec 100644 --- a/pytransifex/config.py +++ b/pytransifex/config.py @@ -33,8 +33,8 @@ def from_env(cls) -> "ApiConfig": path_keys = ["input_directory", "output_directory", "config_file"] mandatory = ["organization_slug", "project_slug"] defaults = { - "output_directory": Path.cwd().joinpath("downloaded_files"), - "config_file": Path.cwd().joinpath(".pytx_config.yaml"), + "output_directory": Path.cwd().joinpath("output"), + "config_file": Path.cwd().joinpath(".pytx_config.yml"), } diff --git a/tests/test_resource_fr.po b/tests/input/test_resource_fr.po similarity index 100% rename from tests/test_resource_fr.po rename to tests/input/test_resource_fr.po diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 57813e1..5a0767c 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -1,3 +1,4 @@ +from shutil import rmtree import unittest from pathlib import Path @@ -13,8 +14,8 @@ def setUpClass(cls): cls.project_name = "Test Project PyTransifex" cls.resource_slug = "test_resource_fr" cls.resource_name = "Test Resource FR" - cls.path_to_file = Path(Path.cwd()).joinpath("tests", "test_resource_fr.po") - cls.output_dir = Path(Path.cwd()).joinpath("tests", "output") + cls.path_to_file = Path.cwd().joinpath("tests", "input", "test_resource_fr.po") + cls.output_dir = Path.cwd().joinpath("tests", "output") if project := cls.tx.get_project(project_slug=cls.project_slug): print("Found old project, removing.") @@ -25,6 +26,11 @@ def setUpClass(cls): project_name=cls.project_name, project_slug=cls.project_slug, private=True ) + @classmethod + def tearDownClass(cls): + if Path.exists(cls.output_dir): + rmtree(cls.output_dir) + def test1_new_api_satisfies_abc(self): assert isinstance(self.tx, Tx) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9824391..1c93c8b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -5,19 +5,19 @@ from pytransifex.api_new import Transifex from pytransifex.cli import cli -from pytransifex.config import CliSettings, defaults class TestCli(unittest.TestCase): @classmethod def setUpClass(cls): - """ cls.tx = Transifex(defer_login=True) cls.project_slug = "test_project_pytransifex" cls.project_name = "Test Project PyTransifex" cls.resource_slug = "test_resource_fr" cls.resource_name = "Test Resource FR" - cls.path_to_file = Path(Path.cwd()).joinpath("tests", "test_resource_fr.po") + cls.path_to_input_dir = Path.cwd().joinpath("tests", "input") + cls.path_to_file = cls.path_to_input_dir.joinpath("test_resource_fr.po") + cls.output_dir = Path.cwd().joinpath("tests", "output") if project := cls.tx.get_project(project_slug=cls.project_slug): print("Found old project, removing.") @@ -27,33 +27,32 @@ def setUpClass(cls): cls.tx.create_project( project_name=cls.project_name, project_slug=cls.project_slug, private=True ) - """ + cls.runner = CliRunner() @classmethod def tearDownClass(cls): - if Path.exists(defaults["config_file"]): - remove(defaults["config_file"]) + if Path.exists(cls.output_dir): + remove(cls.output_dir) def test1_init(self): result = self.runner.invoke( cli, - ["init", "-org", "ORGA", "-p", "PROJECT"], + ["init", "-p", self.project_slug, "-org", "test_pytransifex"], ) passed = result.exit_code == 0 - print(result.output if passed else result) assert passed def test2_push(self): - result = self.runner.invoke(cli, ["push", "-in", "SOME_DIRECTORY"]) + result = self.runner.invoke(cli, ["push", "-in", str(self.path_to_input_dir)]) passed = result.exit_code == 0 and "No such file" in result.output - print(result.output if passed else result) + print(result.output) assert passed def test3_pull(self): - result = self.runner.invoke(cli, ["pull", "-l", "de,fr,it,en"]) + result = self.runner.invoke(cli, ["pull", "-l", "fr_CH,en_GB"]) passed = result.exit_code == 0 - print(result.output if passed else result) + print(result.output) assert passed From 760dfefc77dd43d9a96c4e0aec1f83de1d87c19e Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 1 Dec 2022 09:18:26 +0100 Subject: [PATCH 16/34] Added pre-commit, workflow for pytransifex --- .github/workflows/pre-commit.yaml | 13 +++++++++++ .github/workflows/test_pytx.yml | 17 +++++++++++++++ .pre-commit-config.yaml | 36 +++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .github/workflows/test_pytx.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..4992dc1 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,13 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [master] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: pre-commit/action@v2.0.2 diff --git a/.github/workflows/test_pytx.yml b/.github/workflows/test_pytx.yml new file mode 100644 index 0000000..9b5ab4b --- /dev/null +++ b/.github/workflows/test_pytx.yml @@ -0,0 +1,17 @@ +name: Test Pytx + +on: + pull_request: + push: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Setup CI tests + run: | + pip install requirements.txt + python -m unitest tests -v diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a704bf2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,36 @@ +repos: + # Fix end of files + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: mixed-line-ending + args: + - '--fix=lf' + + # Remove unused imports/variables + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + args: + - "--in-place" + - "--remove-all-unused-imports" + - "--remove-unused-variable" + - "--ignore-init-module-imports" + + # Sort imports + - repo: https://github.com/pycqa/isort + rev: "5.7.0" + hooks: + - id: isort + args: + - --profile + - black + + # Black formatting + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black From b9f1fdf84f6b6f3a10f34f88c3d2b9d32fe9c3d8 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 1 Dec 2022 09:18:59 +0100 Subject: [PATCH 17/34] After running 'pre-commit'. --- .env.example | 2 +- .travis.yml | 2 - README.md | 2 - docs/Makefile | 2 +- docs/build_deploy.sh | 2 +- docs/conf.py | 74 ++++++++++++++++++--------------- pytransifex/__init__.py | 2 - pytransifex/api_new.py | 2 +- pytransifex/api_old.py | 6 +-- pytransifex/config.py | 6 ++- pytransifex/exceptions.py | 4 +- pytransifex/interfaces.py | 2 +- requirements.txt | 2 +- setup.py | 3 +- tests/input/test_resource_fr.po | 2 +- tests/test_api_new.py | 2 +- tests/test_api_old.py | 4 +- tests/test_cli.py | 7 ++-- 18 files changed, 66 insertions(+), 60 deletions(-) diff --git a/.env.example b/.env.example index cfda38d..111e16a 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,2 @@ TX_TOKEN=0123456789 -ORGANIZATION=test_pytransifex \ No newline at end of file +ORGANIZATION=test_pytransifex diff --git a/.travis.yml b/.travis.yml index d004b46..38b6210 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,5 +36,3 @@ deploy: on: branch: master tags: true - - diff --git a/README.md b/README.md index 1fd2f7e..e33e987 100644 --- a/README.md +++ b/README.md @@ -32,5 +32,3 @@ Once installed the package can be run as a CLI; example: pytx pull name -l fr Run `pytx --help` for more information. - - diff --git a/docs/Makefile b/docs/Makefile index 298ea9e..5128596 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -16,4 +16,4 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/build_deploy.sh b/docs/build_deploy.sh index ecdb00d..9a29ae5 100755 --- a/docs/build_deploy.sh +++ b/docs/build_deploy.sh @@ -6,4 +6,4 @@ echo "building docs" pushd docs sed -i "s/__VERSION__/${TRAVIS_TAG}/" conf.py make html -popd \ No newline at end of file +popd diff --git a/docs/conf.py b/docs/conf.py index 1ed61a6..def4379 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,27 +14,28 @@ # import os import sys -sys.path.insert(0, os.path.abspath('..')) + +sys.path.insert(0, os.path.abspath("..")) # -- Project information ----------------------------------------------------- -project = 'pytransifex' -copyright = '2019, Denis Rouzaud' -author = 'Denis Rouzaud' +project = "pytransifex" +copyright = "2019, Denis Rouzaud" +author = "Denis Rouzaud" # The short X.Y version -version = '__VERSION__' +version = "__VERSION__" # The full version, including alpha/beta/rc tags -release = '__VERSION__' +release = "__VERSION__" # -- General configuration --------------------------------------------------- autoclass_content = "class" autodoc_default_flags = [ - "members", - "show-inheritance", + "members", + "show-inheritance", ] autosummary_generate = True # Make _autosummary files and include them napoleon_numpy_docstring = True # Force consistency, leave only Google @@ -44,25 +45,25 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'sphinx.ext.githubpages', - 'sphinx.ext.napoleon', - 'sphinx.ext.autosummary' + "sphinx.ext.autodoc", + "sphinx.ext.coverage", + "sphinx.ext.viewcode", + "sphinx.ext.githubpages", + "sphinx.ext.napoleon", + "sphinx.ext.autosummary", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The master toctree document. -master_doc = 'index' +master_doc = "index" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -74,7 +75,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] # The name of the Pygments (syntax highlighting) style to use. pygments_style = None @@ -85,7 +86,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -96,7 +97,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Custom sidebar templates, must be a dictionary that maps document names # to template names. @@ -112,7 +113,7 @@ # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'pytransifexdoc' +htmlhelp_basename = "pytransifexdoc" # -- Options for LaTeX output ------------------------------------------------ @@ -121,15 +122,12 @@ # The paper size ('letterpaper' or 'a4paper'). # # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # # 'preamble': '', - # Latex figure (float) alignment # # 'figure_align': 'htbp', @@ -139,8 +137,13 @@ # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'pytransifex.tex', 'pytransifex Documentation', - 'Denis Rouzaud', 'manual'), + ( + master_doc, + "pytransifex.tex", + "pytransifex Documentation", + "Denis Rouzaud", + "manual", + ), ] @@ -148,10 +151,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'pytransifex', 'pytransifex Documentation', - [author], 1) -] +man_pages = [(master_doc, "pytransifex", "pytransifex Documentation", [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -160,9 +160,15 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'pytransifex', 'pytransifex Documentation', - author, 'pytransifex', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "pytransifex", + "pytransifex Documentation", + author, + "pytransifex", + "One line description of project.", + "Miscellaneous", + ), ] @@ -181,7 +187,7 @@ # epub_uid = '' # A list of files that should not be packed into the epub file. -epub_exclude_files = ['search.html'] +epub_exclude_files = ["search.html"] # -- Extension configuration ------------------------------------------------- diff --git a/pytransifex/__init__.py b/pytransifex/__init__.py index 139597f..e69de29 100755 --- a/pytransifex/__init__.py +++ b/pytransifex/__init__.py @@ -1,2 +0,0 @@ - - diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index c4e909c..e37118c 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -288,7 +288,7 @@ def push( tuple([project_slug, res, path]) for res, path in zip(resource_slugs, path_to_files) ] - + concurrently( fn=self.update_source_translation, args=args, diff --git a/pytransifex/api_old.py b/pytransifex/api_old.py index cf62525..1c09d44 100755 --- a/pytransifex/api_old.py +++ b/pytransifex/api_old.py @@ -1,9 +1,9 @@ -import os import codecs -import requests import json +import os from typing import Any -from types import FunctionType + +import requests from pytransifex.config import ApiConfig from pytransifex.exceptions import PyTransifexException diff --git a/pytransifex/config.py b/pytransifex/config.py index 004f2ec..a8da826 100644 --- a/pytransifex/config.py +++ b/pytransifex/config.py @@ -80,7 +80,11 @@ def to_disk(self): toml.dump(self.serialize(), fh) def serialize(self) -> dict[str, Any]: - return {k: (str(v) if k in path_keys else v) for k, v in asdict(self).items() if v is not None} + return { + k: (str(v) if k in path_keys else v) + for k, v in asdict(self).items() + if v is not None + } @staticmethod def deserialize(d: dict[str, Any]) -> dict[str, Any]: diff --git a/pytransifex/exceptions.py b/pytransifex/exceptions.py index 61a97b6..2bad48b 100755 --- a/pytransifex/exceptions.py +++ b/pytransifex/exceptions.py @@ -10,10 +10,10 @@ def __init__(self, response=None): def __str__(self): if self.response is None: return super(PyTransifexException, self).__str__() - return '{code} from {url}: {content}'.format( + return "{code} from {url}: {content}".format( code=self.response.status_code, url=self.response.url, - content=self.response.content + content=self.response.content, ) diff --git a/pytransifex/interfaces.py b/pytransifex/interfaces.py index 7f8c61a..562ebf9 100644 --- a/pytransifex/interfaces.py +++ b/pytransifex/interfaces.py @@ -1,5 +1,5 @@ +from abc import ABC, abstractmethod from typing import Any -from abc import abstractmethod, ABC class Tx(ABC): diff --git a/requirements.txt b/requirements.txt index 70e5dc5..ea36068 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,4 @@ nose2 transifex-python click python-dotenv -toml \ No newline at end of file +toml diff --git a/setup.py b/setup.py index b1ca7c3..bac75ad 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ -from setuptools import setup import sys +from setuptools import setup + python_min_version = (3, 10) if sys.version_info < python_min_version: diff --git a/tests/input/test_resource_fr.po b/tests/input/test_resource_fr.po index 7704ba3..94c0386 100644 --- a/tests/input/test_resource_fr.po +++ b/tests/input/test_resource_fr.po @@ -23,4 +23,4 @@ msgstr "Willkommen zurück, %1$s! Dein letzter Besuch war am %2$s" msgid "%d page read." msgid_plural "%d pages read." msgstr[0] "Eine Seite gelesen wurde." -msgstr[1] "%d Seiten gelesen wurden." \ No newline at end of file +msgstr[1] "%d Seiten gelesen wurden." diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 5a0767c..155ed63 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -1,6 +1,6 @@ -from shutil import rmtree import unittest from pathlib import Path +from shutil import rmtree from pytransifex.api_new import Transifex from pytransifex.interfaces import Tx diff --git a/tests/test_api_old.py b/tests/test_api_old.py index 282cc66..4a9f174 100644 --- a/tests/test_api_old.py +++ b/tests/test_api_old.py @@ -1,11 +1,11 @@ #! /usr/bin/env python -import unittest import os +import unittest +from pytransifex.api_old import Transifex from pytransifex.config import ApiConfig from pytransifex.exceptions import PyTransifexException -from pytransifex.api_old import Transifex class TestTranslation(unittest.TestCase): diff --git a/tests/test_cli.py b/tests/test_cli.py index 1c93c8b..6cd7c88 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,7 @@ import unittest -from pathlib import Path from os import remove +from pathlib import Path + from click.testing import CliRunner from pytransifex.api_new import Transifex @@ -15,7 +16,7 @@ def setUpClass(cls): cls.project_name = "Test Project PyTransifex" cls.resource_slug = "test_resource_fr" cls.resource_name = "Test Resource FR" - cls.path_to_input_dir = Path.cwd().joinpath("tests", "input") + cls.path_to_input_dir = Path.cwd().joinpath("tests", "input") cls.path_to_file = cls.path_to_input_dir.joinpath("test_resource_fr.po") cls.output_dir = Path.cwd().joinpath("tests", "output") @@ -27,7 +28,7 @@ def setUpClass(cls): cls.tx.create_project( project_name=cls.project_name, project_slug=cls.project_slug, private=True ) - + cls.runner = CliRunner() @classmethod From ec42fc4b2a27aed31a8dc01daef2f22cdd45d445 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Mon, 19 Dec 2022 09:46:37 +0100 Subject: [PATCH 18/34] Proper loggin instead of print --- pytransifex/__init__.py | 3 + pytransifex/api_new.py | 27 +-- pytransifex/api_old.py | 494 ---------------------------------------- 3 files changed, 17 insertions(+), 507 deletions(-) delete mode 100755 pytransifex/api_old.py diff --git a/pytransifex/__init__.py b/pytransifex/__init__.py index e69de29..ca0e42e 100755 --- a/pytransifex/__init__.py +++ b/pytransifex/__init__.py @@ -0,0 +1,3 @@ +import logging + +logging.basicConfig(level=logging.INFO) diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index e37118c..28b2bdb 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -1,3 +1,4 @@ +import logging from os import mkdir from pathlib import Path from typing import Any @@ -42,7 +43,7 @@ def login(self): organization = tx_api.Organization.get(slug=self.organization_name) self.projects = organization.fetch("projects") self.organization = organization - print(f"Logged in as organization: {self.organization_name}") + logging.info(f"Logged in as organization: {self.organization_name}") @ensure_login def create_project( @@ -65,7 +66,7 @@ def create_project( private=private, organization=self.organization, ) - print("Project created!") + logging.info("Project created!") return res except JsonApiException as error: if "already exists" in error.detail: @@ -77,7 +78,7 @@ def get_project(self, project_slug: str) -> None | Resource: if self.projects: try: res = self.projects.get(slug=project_slug) - print("Got the project!") + logging.info("Got the project!") return res except DoesNotExist: return None @@ -87,7 +88,7 @@ def list_resources(self, project_slug: str) -> list[Resource]: """List all resources for the project passed as argument""" if self.projects: res = self.projects.filter(slug=project_slug) - print("Obtained these resources") + logging.info("Obtained these resources:") return res raise Exception( f"Unable to find any project under this organization: '{self.organization}'" @@ -103,7 +104,7 @@ def create_resource( ): """Create a resource using the given file contents, slugs and names""" if not (resource_slug or resource_name): - raise Exception("Please give either a resource_slug or resource_name") + raise Exception("Please give either a resource_slug or a resource_name") if project := self.get_project(project_slug=project_slug): resource = tx_api.Resource.create( @@ -116,7 +117,7 @@ def create_resource( with open(path_to_file, "r") as fh: content = fh.read() tx_api.ResourceStringsAsyncUpload.upload(content, resource=resource) - print(f"Resource created: {resource_slug or resource_name}") + logging.info(f"Resource created: {resource_slug or resource_name}") else: raise Exception( f"Not project could be found wiht the slug '{project_slug}'. Please create a project first." @@ -143,7 +144,7 @@ def update_source_translation( tx_api.ResourceStringsAsyncUpload.upload( content, resource=resource ) - print(f"Source updated for resource: {resource_slug}") + logging.info(f"Source updated for resource: {resource_slug}") return raise Exception( @@ -178,7 +179,7 @@ def get_translation( with open(file_name, "w") as fh: fh.write(translated_content) - print( + logging.info( f"Translations downloaded and written to file (resource: {resource_slug})" ) @@ -204,7 +205,7 @@ def list_languages(self, project_slug: str) -> list[Any]: if self.projects: if project := self.projects.get(slug=project_slug): languages = project.fetch("languages") - print(f"Obtained these languages") + logging.info(f"Obtained these languages") return languages raise Exception( f"Unable to find any project with this slug: '{project_slug}'" @@ -227,7 +228,7 @@ def create_language( if coordinators: project.add("coordinators", coordinators) - print(f"Created language resource for {language_code}") + logging.info(f"Created language resource for {language_code}") @ensure_login def project_exists(self, project_slug: str) -> bool: @@ -246,7 +247,7 @@ def ping(self): Exposing this just for the sake of satisfying qgis-plugin-cli's expectations There is no need to ping the server on the current implementation, as connection is handled by the SDK """ - print("'ping' is deprecated!") + logging.info("'ping' is deprecated!") @ensure_login def get_project_stats(self, project_slug: str) -> dict[str, Any]: @@ -269,7 +270,7 @@ def pull( for l_code in language_codes: for slug in resource_slugs: args.append(tuple([project_slug, slug, l_code, output_dir])) - print("ARGS", args) + logging.info("ARGS", args) concurrently( fn=self.get_translation, args=args, @@ -294,7 +295,7 @@ def push( args=args, ) - print(f"Pushes some {len(resource_slugs)} files!") + logging.info(f"Pushes some {len(resource_slugs)} files!") class Transifex: diff --git a/pytransifex/api_old.py b/pytransifex/api_old.py deleted file mode 100755 index 1c09d44..0000000 --- a/pytransifex/api_old.py +++ /dev/null @@ -1,494 +0,0 @@ -import codecs -import json -import os -from typing import Any - -import requests - -from pytransifex.config import ApiConfig -from pytransifex.exceptions import PyTransifexException - - -class Transifex: - def __init__(self, config: ApiConfig): - """ - Initializes Transifex - - Parameters - ---------- - api_token - the API token to the service - organization - the name of the organization - i18n_type - the type of translation (PO, QT, …) - defaults to: PO - """ - self.auth = ("api", config.api_token) - self.api_key = config.api_token - self.organization = config.organization - self.i18n_type = config.i18n_type - - def create_project( - self, - project_slug: str, - project_name: str | None = None, - source_language_code: str = "en-gb", - outsource_project_name: str | None = None, - private: bool = False, - repository_url: str | None = None, - ): - """ - Create a new project on Transifex - - Parameters - ---------- - project_slug - the project slug - project_name - the project name, defaults to the project slug - source_language_code - the source language code, defaults to 'en-gb' - outsource_project_name - the name of the project to outsource translation team management to - private - controls if this is created as a closed source or open-source - project, defaults to `False` - repository_url - The url for the repository. This is required if private is set to - False - - Raises - ------ - `PyTransifexException` - if project was not created properly - """ - if project_name is None: - project_name = project_slug - - url = "https://rest.api.transifex.com/projects" - data: dict[str, Any] = { - "data": { - "attributes": { - "name": project_name, - "slug": project_slug, - "description": project_name, - "private": private, - "repository_url": repository_url, - }, - "relationships": { - "organization": { - "data": { - "id": "o:{}".format(self.organization), - "type": "organizations", - } - }, - "source_language": { - "data": { - "id": "l:{}".format(source_language_code), - "type": "languages", - } - }, - }, - "type": "projects", - } - } - if outsource_project_name is not None: - data["outsource"] = outsource_project_name - - response = requests.post( - url, - data=json.dumps(data), - headers={ - "Content-Type": "application/vnd.api+json", - "Authorization": "Bearer {}".format(self.api_key), - }, - ) - - if response.status_code != requests.codes["OK"]: - print("Could not create project with data: {}".format(data)) - raise PyTransifexException(response) - - def delete_project(self, project_slug: str): - """ - Deletes the project - - Parameters - ---------- - project_slug - the project slug - - Raises - ------ - `PyTransifexException` - """ - url = "https://rest.api.transifex.com/projects/o:{o}:p:{p}".format( - o=self.organization, p=project_slug - ) - response = requests.delete( - url, - headers={ - "Content-Type": "application/vnd.api+json", - "Authorization": "Bearer {}".format(self.api_key), - }, - ) - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - - def list_resources(self, project_slug) -> list: - """ - List all resources in a project - - Parameters - ---------- - project_slug - the project slug - - Returns - ------- - list of dictionaries with resources info - each dictionary may contain - category - i18n_type - source_language_code - slug - name - - Raises - ------ - `PyTransifexException` - """ - url = "https://api.transifex.com/organizations/{o}/projects/{p}/resources/".format( - o=self.organization, p=project_slug - ) - response = requests.get(url, auth=self.auth) - - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - - return json.loads(codecs.decode(response.content, "utf-8")) - - def delete_team(self, team_slug: str): - url = "https://rest.api.transifex.com/teams/o:{o}:t:{t}".format( - o=self.organization, t=team_slug - ) - response = requests.delete( - url, - headers={ - "Content-Type": "application/vnd.api+json", - "Authorization": "Bearer {}".format(self.api_key), - }, - ) - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - - def create_resource( - self, project_slug, path_to_file, resource_slug=None, resource_name=None - ): - """ - Creates a new resource with the specified slug from the given file. - - Parameters - ---------- - project_slug - the project slug - path_to_file - the path to the file which will be uploaded - resource_slug - the resource slug, defaults to a sluggified version of the filename - resource_name - the resource name, defaults to the resource name - - Raises - ------ - `PyTransifexException` - `IOError` - """ - url = "https://www.transifex.com/api/2/project/{p}/resources/".format( - p=project_slug - ) - data = { - "name": resource_name or resource_slug, - "slug": resource_slug, - "i18n_type": self.i18n_type, - "content": open(path_to_file, "r").read(), - } - response = requests.post( - url, - data=json.dumps(data), - auth=self.auth, - headers={"content-type": "application/json"}, - ) - - if response.status_code != requests.codes["CREATED"]: - raise PyTransifexException(response) - - def delete_resource(self, project_slug, resource_slug): - """ - Deletes the given resource - - Parameters - ---------- - project_slug - the project slug - resource_slug - the resource slug - - Raises - ------ - `PyTransifexException` - """ - url = "{u}/project/{s}/resource/{r}".format( - u=self._base_api_url, s=project_slug, r=resource_slug - ) - response = requests.delete(url, auth=self.auth) - if response.status_code != requests.codes["NO_CONTENT"]: - raise PyTransifexException(response) - - def update_source_translation(self, project_slug, resource_slug, path_to_file): - """ - Update the source translation for a give resource - - Parameters - ---------- - project_slug - the project slug - resource_slug - the resource slug - path_to_file - the path to the file which will be uploaded - - Returns - ------- - dictionary with info - Info may include keys - strings_added - strings_updated - redirect - - Raises - ------ - `PyTransifexException` - `IOError` - """ - url = "https://www.transifex.com/api/2/project/{s}/resource/{r}/content".format( - s=project_slug, r=resource_slug - ) - content = open(path_to_file, "r").read() - data = {"content": content} - response = requests.put( - url, - data=json.dumps(data), - auth=self.auth, - headers={"content-type": "application/json"}, - ) - - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - else: - return json.loads(codecs.decode(response.content, "utf-8")) - - def create_translation( - self, project_slug, resource_slug, language_code, path_to_file - ) -> dict: - """ - Creates or updates the translation for the specified language - - Parameters - ---------- - project_slug - the project slug - resource_slug - the resource slug - language_code - the language_code of the file - path_to_file - the path to the file which will be uploaded - - Returns - ------- - dictionary with info - Info may include keys - strings_added - strings_updated - redirect - - Raises - ------ - `PyTransifexException` - `IOError` - """ - url = "https://www.transifex.com/api/2/project/{s}/resource/{r}/translation/{l}".format( - s=project_slug, r=resource_slug, l=language_code - ) - content = open(path_to_file, "r").read() - data = {"content": content} - response = requests.put( - url, - data=json.dumps(data), - auth=self.auth, - headers={"content-type": "application/json"}, - ) - - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - else: - return json.loads(codecs.decode(response.content, "utf-8")) - - def get_translation( - self, - project_slug: str, - resource_slug: str, - language_code: str, - path_to_file: str, - ): - """ - Returns the requested translation, if it exists. The translation is - returned as a serialized string, unless the GET parameter file is - specified. - - Parameters - ---------- - project_slug - The project slug - resource_slug - The resource slug - language_code - The language_code of the file. - This should be the *Transifex* language code - path_to_file - The path to the translation file which will be saved. - If the directory does not exist, it will be automatically created. - - Raises - ------ - `PyTransifexException` - `IOError` - """ - url = "https://www.transifex.com/api/2/project/{s}/resource/{r}/translation/{l}".format( - s=project_slug, r=resource_slug, l=language_code - ) - query = {"file": ""} - response = requests.get(url, auth=self.auth, params=query) - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - else: - os.makedirs(os.path.dirname(path_to_file), exist_ok=True) - with open(path_to_file, "wb") as f: - for line in response.iter_content(): - f.write(line) - - def list_languages(self, project_slug, resource_slug): - """ - List all the languages available for a given resource in a project - - Parameters - ---------- - project_slug - The project slug - resource_slug - The resource slug - - Returns - ------- - list - The language codes which this resource has translations - - Raises - ------ - `PyTransifexException` - """ - url = "https://www.transifex.com/api/2/project/{s}/resource/{r}".format( - s=project_slug, r=resource_slug - ) - response = requests.get(url, auth=self.auth, params={"details": ""}) - - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - - content = json.loads(codecs.decode(response.content, "utf-8")) - languages = [language["code"] for language in content["available_languages"]] - return languages - - def create_language( - self, project_slug: str, language_code: str, coordinators: list - ): - """ - Create a new language for the given project - Parameters - ---------- - project_slug: - language_code: - coordinators: - list of coordinators - """ - url = "https://www.transifex.com/api/2/project/{s}/languages".format( - s=project_slug - ) - data = {"language_code": language_code, "coordinators": coordinators} - response = requests.post( - url, - headers={"content-type": "application/json"}, - auth=self.auth, - data=json.dumps(data), - ) - if response.status_code != requests.codes["CREATED"]: - raise PyTransifexException(response) - - def coordinator(self, project_slug: str, language_code: str = "en") -> str: - """ - Return the coordinator of the the project - - Parameters - ---------- - project_slug: - language_code: - """ - url = "https://www.transifex.com/api/2/project/{s}/language/{l}/coordinators".format( - s=project_slug, l=language_code - ) - response = requests.get(url, auth=self.auth) - if response.status_code != requests.codes["OK"]: - raise PyTransifexException(response) - content = json.loads(codecs.decode(response.content, "utf-8")) - return content["coordinators"] - - def project_exists(self, project_slug) -> bool: - """ - Check if there is a project with the given slug registered with - Transifex - - Parameters - ---------- - project_slug - The project slug - """ - url = "https://rest.api.transifex.com/projects/o:{o}:p:{s}".format( - o=self.organization, s=project_slug - ) - response = requests.get( - url, - headers={ - "Content-Type": "application/vnd.api+json", - "Authorization": "Bearer {}".format(self.api_key), - }, - ) - if response.status_code == requests.codes["OK"]: - return True - elif response.status_code == requests.codes["NOT_FOUND"]: - return False - else: - raise PyTransifexException(response) - - def ping(self) -> bool: - """ - Check the connection to the server and the auth credentials - """ - url = "https://api.transifex.com/organizations/{}/projects/".format( - self.organization - ) - response = requests.get(url, auth=self.auth) - success = response.status_code == requests.codes["OK"] - if not success: - raise PyTransifexException(response) - return success From 42d060911d0cd2325d6a2d9176c3f8dfa9196908 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Mon, 19 Dec 2022 09:52:18 +0100 Subject: [PATCH 19/34] Weakened version build requirements --- .github/workflows/test_pytx.yml | 4 +-- pytransifex/api_new.py | 8 +++--- setup.py | 2 +- tests/test_api_old.py | 46 --------------------------------- 4 files changed, 7 insertions(+), 53 deletions(-) delete mode 100644 tests/test_api_old.py diff --git a/.github/workflows/test_pytx.yml b/.github/workflows/test_pytx.yml index 9b5ab4b..f3fd6c3 100644 --- a/.github/workflows/test_pytx.yml +++ b/.github/workflows/test_pytx.yml @@ -13,5 +13,5 @@ jobs: - name: Setup CI tests run: | - pip install requirements.txt - python -m unitest tests -v + pip install -r requirements.txt + python -m unittest discover -s ./tests -p 'test_*.py' diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 28b2bdb..2443744 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -52,10 +52,10 @@ def create_project( project_name: str | None = None, source_language_code: str = "en_GB", private: bool = False, - *args, - **kwargs, + *args, # absorbing extra args + **kwargs, # absorbing extra kwargs ) -> None | Resource: - """Create a project. args, kwargs are there to absorb unnecessary arguments from consumers.""" + """Create a project.""" source_language = tx_api.Language.get(code=source_language_code) try: @@ -69,7 +69,7 @@ def create_project( logging.info("Project created!") return res except JsonApiException as error: - if "already exists" in error.detail: + if "already exists" in error.detail: # type: ignore return self.get_project(project_slug=project_slug) @ensure_login diff --git a/setup.py b/setup.py index bac75ad..a867238 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup -python_min_version = (3, 10) +python_min_version = (3, 8) if sys.version_info < python_min_version: sys.exit( diff --git a/tests/test_api_old.py b/tests/test_api_old.py deleted file mode 100644 index 4a9f174..0000000 --- a/tests/test_api_old.py +++ /dev/null @@ -1,46 +0,0 @@ -#! /usr/bin/env python - -import os -import unittest - -from pytransifex.api_old import Transifex -from pytransifex.config import ApiConfig -from pytransifex.exceptions import PyTransifexException - - -class TestTranslation(unittest.TestCase): - def setUp(self): - token = os.getenv("TX_TOKEN") - assert token is not None - self.t = Transifex( - ApiConfig(organization="pytransifex", api_token=token, i18n_type="PO") - ) - self.project_slug = "pytransifex-test-project" - self.project_name = "PyTransifex Test project" - self.source_lang = "fr_FR" - - def tearDown(self): - try: - self.t.delete_project(self.project_slug) - except PyTransifexException: - pass - try: - self.t.delete_team("{}-team".format(self.project_slug)) - except PyTransifexException: - pass - - def test_creation(self): - self.tearDown() - self.t.create_project( - project_name=self.project_name, - project_slug=self.project_slug, - source_language_code=self.source_lang, - private=False, - repository_url="https://www.github.com/opengisch/pytransifex", - ) - - self.assertTrue(self.t.project_exists(self.project_slug)) - - -if __name__ == "__main__": - unittest.main() From e625eb4538583a41738eafa5c0eea54d770aff63 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 22 Dec 2022 15:49:03 +0100 Subject: [PATCH 20/34] PluginManager for managing plugins registed by the user to generate custom Transifex configurations. --- pytransifex/api_new.py | 2 +- .../config-plugins/opengis-mkdocs/main.py | 56 ++++++++++++++ .../opengis-mkdocs/requirements.txt | 1 + pytransifex/plugins_manager.py | 74 +++++++++++++++++++ tests/test_config-plugins.py | 16 ++++ 5 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 pytransifex/config-plugins/opengis-mkdocs/main.py create mode 100644 pytransifex/config-plugins/opengis-mkdocs/requirements.txt create mode 100644 pytransifex/plugins_manager.py create mode 100644 tests/test_config-plugins.py diff --git a/pytransifex/api_new.py b/pytransifex/api_new.py index 2443744..be09c1b 100644 --- a/pytransifex/api_new.py +++ b/pytransifex/api_new.py @@ -69,7 +69,7 @@ def create_project( logging.info("Project created!") return res except JsonApiException as error: - if "already exists" in error.detail: # type: ignore + if "already exists" in error.detail: # type: ignore return self.get_project(project_slug=project_slug) @ensure_login diff --git a/pytransifex/config-plugins/opengis-mkdocs/main.py b/pytransifex/config-plugins/opengis-mkdocs/main.py new file mode 100644 index 0000000..609a6b2 --- /dev/null +++ b/pytransifex/config-plugins/opengis-mkdocs/main.py @@ -0,0 +1,56 @@ +# © 2022 Mario Baranzini @ mario.baranzini@opengis.ch +import glob +import logging +import os +from typing import NamedTuple + +import frontmatter + + +class TxProjectConfig(NamedTuple): + TX_ORGANIZATION = "opengisch" + TX_PROJECT = "qfield-documentation" + TX_SOURCE_LANG = "en" + TX_TYPE = "GITHUBMARKDOWN" + + +def create_transifex_config(config: TxProjectConfig): + """Parse all source documentation files and add the ones with tx_slug metadata + defined to transifex config file. + """ + logging.info("Start creating transifex configuration") + + current_dir = os.path.dirname(os.path.abspath(__file__)) + config_file = os.path.join(current_dir, "..", ".tx", "config") + root = os.path.join(current_dir, "..") + count = 0 + + with open(config_file, "w") as f: + f.write("[main]\n") + f.write("host = https://www.transifex.com\n\n") + + for file in glob.iglob( + current_dir + "/../documentation/**/*.en.md", recursive=True + ): + + # Get relative path of file + relative_path = os.path.relpath(file, start=root) + + tx_slug = frontmatter.load(file).get("tx_slug", None) + + if tx_slug: + logging.info( + f"Found file with tx_slug defined: {relative_path}, {tx_slug}" + ) + f.write( + f"[o:{config.TX_ORGANIZATION}:p:{config.TX_PROJECT}:r:{tx_slug}]\n" + ) + f.write( + f"file_filter = {''.join(relative_path.split('.')[:-2])}..md\n" + ) + f.write(f"source_file = {relative_path}\n") + f.write(f"source_lang = {config.TX_SOURCE_LANG}\n") + f.write(f"type = {config.TX_TYPE}\n\n") + count += 1 + + logging.info(f"Transifex configuration created. {count} resources added.") diff --git a/pytransifex/config-plugins/opengis-mkdocs/requirements.txt b/pytransifex/config-plugins/opengis-mkdocs/requirements.txt new file mode 100644 index 0000000..aad499a --- /dev/null +++ b/pytransifex/config-plugins/opengis-mkdocs/requirements.txt @@ -0,0 +1 @@ +python-frontmatter==1.0.0 diff --git a/pytransifex/plugins_manager.py b/pytransifex/plugins_manager.py new file mode 100644 index 0000000..ef4054f --- /dev/null +++ b/pytransifex/plugins_manager.py @@ -0,0 +1,74 @@ +import sys +from functools import reduce +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from typing import Any + + +class PluginManager: + # list of path_to_ subdir, path_to_ main.py + discovered_subdir_main: list[tuple[Path, Path]] = [] + # successfully imported modules + imported_modules: dict[str, Any] = {} + + @staticmethod + def discover(): + """ + Expecting directory structure: + pytransifex/ + config-plugins/{plugins} + main.py + """ + plugins_dir = Path.cwd().joinpath("pytransifex", "config-plugins") + subdirs = [f for f in plugins_dir.iterdir() if f.is_dir()] + + def collect_mains(acc: list[Path], sub: Path): + main = Path(sub).joinpath("main.py") + + if main.exists() and main.is_file(): + acc.append(main) + + return acc + + collected = reduce(collect_mains, subdirs, []) + PluginManager.discovered_subdir_main = list(zip(subdirs, collected)) + + @staticmethod + def load_plugin(target_plugin_dir: str): + conventional_main = "main.py" + + if name_main := next( + ( + (main_p, directory_p.name) + for directory_p, main_p in PluginManager.discovered_subdir_main + if directory_p.name == target_plugin_dir + ), + None, + ): + main, name = name_main + + if name in PluginManager.imported_modules: + print("Already imported! Nothing to do.") + return + + if spec := spec_from_file_location(conventional_main, main): + module = module_from_spec(spec) + sys.modules[conventional_main] = module + + if spec.loader: + spec.loader.exec_module(module) + PluginManager.imported_modules[name] = module + print( + f"Successfully imported and loaded {module}! Imported modules read: {PluginManager.imported_modules}" + ) + + else: + raise Exception(f"Failed to load module '{module}' at {main}") + + else: + raise Exception(f"Unable to find spec 'main.py' for {main}") + + else: + raise Exception( + f"Couldn't find the '{target_plugin_dir}' directory; it was expected to be a child of the 'pytransifex' directory." + ) diff --git a/tests/test_config-plugins.py b/tests/test_config-plugins.py new file mode 100644 index 0000000..720a7fc --- /dev/null +++ b/tests/test_config-plugins.py @@ -0,0 +1,16 @@ +import unittest + +from pytransifex.plugins_manager import PluginManager + + +class TestPluginsRegistrar(unittest.TestCase): + def test1_plugin_discover(self): + PluginManager.discover() + assert list(PluginManager.discovered_subdir_main) + + def test2_plugin_import(self): + PluginManager.load_plugin("opengis-mkdocs") + assert list(PluginManager.imported_modules) + + def test3_imported_plugin(self): + assert PluginManager.imported_modules["opengis-mkdocs"].TxProjectConfig From bcbc0dcfc8dcddcf129f54a1186483659a3fa729 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 14 Feb 2023 08:53:26 +0100 Subject: [PATCH 21/34] Cleanup --- pytransifex/{api_new.py => api.py} | 0 pytransifex/cli.py | 2 +- .../config-plugins/opengis-mkdocs/main.py | 56 -------------- .../opengis-mkdocs/requirements.txt | 1 - pytransifex/plugins_manager.py | 74 ------------------- tests/test_api_new.py | 2 +- tests/test_cli.py | 2 +- tests/test_config-plugins.py | 16 ---- 8 files changed, 3 insertions(+), 150 deletions(-) rename pytransifex/{api_new.py => api.py} (100%) delete mode 100644 pytransifex/config-plugins/opengis-mkdocs/main.py delete mode 100644 pytransifex/config-plugins/opengis-mkdocs/requirements.txt delete mode 100644 pytransifex/plugins_manager.py delete mode 100644 tests/test_config-plugins.py diff --git a/pytransifex/api_new.py b/pytransifex/api.py similarity index 100% rename from pytransifex/api_new.py rename to pytransifex/api.py diff --git a/pytransifex/cli.py b/pytransifex/cli.py index fd616e2..d8db4e3 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -3,7 +3,7 @@ import click -from pytransifex.api_new import Transifex +from pytransifex.api import Transifex from pytransifex.config import CliSettings client = Transifex(defer_login=True) diff --git a/pytransifex/config-plugins/opengis-mkdocs/main.py b/pytransifex/config-plugins/opengis-mkdocs/main.py deleted file mode 100644 index 609a6b2..0000000 --- a/pytransifex/config-plugins/opengis-mkdocs/main.py +++ /dev/null @@ -1,56 +0,0 @@ -# © 2022 Mario Baranzini @ mario.baranzini@opengis.ch -import glob -import logging -import os -from typing import NamedTuple - -import frontmatter - - -class TxProjectConfig(NamedTuple): - TX_ORGANIZATION = "opengisch" - TX_PROJECT = "qfield-documentation" - TX_SOURCE_LANG = "en" - TX_TYPE = "GITHUBMARKDOWN" - - -def create_transifex_config(config: TxProjectConfig): - """Parse all source documentation files and add the ones with tx_slug metadata - defined to transifex config file. - """ - logging.info("Start creating transifex configuration") - - current_dir = os.path.dirname(os.path.abspath(__file__)) - config_file = os.path.join(current_dir, "..", ".tx", "config") - root = os.path.join(current_dir, "..") - count = 0 - - with open(config_file, "w") as f: - f.write("[main]\n") - f.write("host = https://www.transifex.com\n\n") - - for file in glob.iglob( - current_dir + "/../documentation/**/*.en.md", recursive=True - ): - - # Get relative path of file - relative_path = os.path.relpath(file, start=root) - - tx_slug = frontmatter.load(file).get("tx_slug", None) - - if tx_slug: - logging.info( - f"Found file with tx_slug defined: {relative_path}, {tx_slug}" - ) - f.write( - f"[o:{config.TX_ORGANIZATION}:p:{config.TX_PROJECT}:r:{tx_slug}]\n" - ) - f.write( - f"file_filter = {''.join(relative_path.split('.')[:-2])}..md\n" - ) - f.write(f"source_file = {relative_path}\n") - f.write(f"source_lang = {config.TX_SOURCE_LANG}\n") - f.write(f"type = {config.TX_TYPE}\n\n") - count += 1 - - logging.info(f"Transifex configuration created. {count} resources added.") diff --git a/pytransifex/config-plugins/opengis-mkdocs/requirements.txt b/pytransifex/config-plugins/opengis-mkdocs/requirements.txt deleted file mode 100644 index aad499a..0000000 --- a/pytransifex/config-plugins/opengis-mkdocs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -python-frontmatter==1.0.0 diff --git a/pytransifex/plugins_manager.py b/pytransifex/plugins_manager.py deleted file mode 100644 index ef4054f..0000000 --- a/pytransifex/plugins_manager.py +++ /dev/null @@ -1,74 +0,0 @@ -import sys -from functools import reduce -from importlib.util import module_from_spec, spec_from_file_location -from pathlib import Path -from typing import Any - - -class PluginManager: - # list of path_to_ subdir, path_to_ main.py - discovered_subdir_main: list[tuple[Path, Path]] = [] - # successfully imported modules - imported_modules: dict[str, Any] = {} - - @staticmethod - def discover(): - """ - Expecting directory structure: - pytransifex/ - config-plugins/{plugins} - main.py - """ - plugins_dir = Path.cwd().joinpath("pytransifex", "config-plugins") - subdirs = [f for f in plugins_dir.iterdir() if f.is_dir()] - - def collect_mains(acc: list[Path], sub: Path): - main = Path(sub).joinpath("main.py") - - if main.exists() and main.is_file(): - acc.append(main) - - return acc - - collected = reduce(collect_mains, subdirs, []) - PluginManager.discovered_subdir_main = list(zip(subdirs, collected)) - - @staticmethod - def load_plugin(target_plugin_dir: str): - conventional_main = "main.py" - - if name_main := next( - ( - (main_p, directory_p.name) - for directory_p, main_p in PluginManager.discovered_subdir_main - if directory_p.name == target_plugin_dir - ), - None, - ): - main, name = name_main - - if name in PluginManager.imported_modules: - print("Already imported! Nothing to do.") - return - - if spec := spec_from_file_location(conventional_main, main): - module = module_from_spec(spec) - sys.modules[conventional_main] = module - - if spec.loader: - spec.loader.exec_module(module) - PluginManager.imported_modules[name] = module - print( - f"Successfully imported and loaded {module}! Imported modules read: {PluginManager.imported_modules}" - ) - - else: - raise Exception(f"Failed to load module '{module}' at {main}") - - else: - raise Exception(f"Unable to find spec 'main.py' for {main}") - - else: - raise Exception( - f"Couldn't find the '{target_plugin_dir}' directory; it was expected to be a child of the 'pytransifex' directory." - ) diff --git a/tests/test_api_new.py b/tests/test_api_new.py index 155ed63..e493ee6 100644 --- a/tests/test_api_new.py +++ b/tests/test_api_new.py @@ -2,7 +2,7 @@ from pathlib import Path from shutil import rmtree -from pytransifex.api_new import Transifex +from pytransifex.api import Transifex from pytransifex.interfaces import Tx diff --git a/tests/test_cli.py b/tests/test_cli.py index 6cd7c88..9a81a9c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,7 +4,7 @@ from click.testing import CliRunner -from pytransifex.api_new import Transifex +from pytransifex.api import Transifex from pytransifex.cli import cli diff --git a/tests/test_config-plugins.py b/tests/test_config-plugins.py deleted file mode 100644 index 720a7fc..0000000 --- a/tests/test_config-plugins.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest - -from pytransifex.plugins_manager import PluginManager - - -class TestPluginsRegistrar(unittest.TestCase): - def test1_plugin_discover(self): - PluginManager.discover() - assert list(PluginManager.discovered_subdir_main) - - def test2_plugin_import(self): - PluginManager.load_plugin("opengis-mkdocs") - assert list(PluginManager.imported_modules) - - def test3_imported_plugin(self): - assert PluginManager.imported_modules["opengis-mkdocs"].TxProjectConfig From 23b63323d4327364f5323886fdc61ca1fb866f59 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 14 Feb 2023 16:29:29 +0100 Subject: [PATCH 22/34] ci envars --- .github/workflows/test_pytx.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_pytx.yml b/.github/workflows/test_pytx.yml index f3fd6c3..edf1cae 100644 --- a/.github/workflows/test_pytx.yml +++ b/.github/workflows/test_pytx.yml @@ -12,6 +12,10 @@ jobs: uses: actions/checkout@v3 - name: Setup CI tests + env: + organization: ${{ secrets.ORGANIZATION }} + tx_token: ${{ secrets.TX_TOKEN }} run: | pip install -r requirements.txt - python -m unittest discover -s ./tests -p 'test_*.py' + TX_TOKEN=$tx_token ORGANIZATION=$organization \ + python -m unittest discover -s ./tests -p 'test_*.py' From 0dc475ed92a5bb2cbd9c50e91a6a4ab3f1c05400 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Tue, 14 Feb 2023 16:56:43 +0100 Subject: [PATCH 23/34] Cleanup --- .github/workflows/test_pytx.yml | 2 +- .pytx_config.yml | 4 ---- tests/{test_api_new.py => test_api.py} | 0 3 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 .pytx_config.yml rename tests/{test_api_new.py => test_api.py} (100%) diff --git a/.github/workflows/test_pytx.yml b/.github/workflows/test_pytx.yml index edf1cae..ffa2110 100644 --- a/.github/workflows/test_pytx.yml +++ b/.github/workflows/test_pytx.yml @@ -11,7 +11,7 @@ jobs: - name: Check out repository code uses: actions/checkout@v3 - - name: Setup CI tests + - name: Run CI tests env: organization: ${{ secrets.ORGANIZATION }} tx_token: ${{ secrets.TX_TOKEN }} diff --git a/.pytx_config.yml b/.pytx_config.yml deleted file mode 100644 index cf86124..0000000 --- a/.pytx_config.yml +++ /dev/null @@ -1,4 +0,0 @@ -organization_slug = "test_pytransifex" -project_slug = "test_project_pytransifex" -output_directory = "/home/hades/opengis/pytransifex/output" -config_file = "/home/hades/opengis/pytransifex/.pytx_config.yml" diff --git a/tests/test_api_new.py b/tests/test_api.py similarity index 100% rename from tests/test_api_new.py rename to tests/test_api.py From a944c0364786c51629c3e32b8a9cfa81fcfbdd62 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 09:26:48 +0100 Subject: [PATCH 24/34] Updated pre-commit config to avoid failure in CI --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a704bf2..b066eb0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: # Sort imports - repo: https://github.com/pycqa/isort - rev: "5.7.0" + rev: "5.12.0" hooks: - id: isort args: From 076e22e23af83e5095261eecb3d7b70d7fa369ce Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 09:33:14 +0100 Subject: [PATCH 25/34] changed CI trigger to: either PR or push --- .github/workflows/pre-commit.yaml | 5 ++++- .github/workflows/test_pytx.yml | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 4992dc1..104d2c4 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -2,8 +2,11 @@ name: pre-commit on: pull_request: + branches: + - master push: - branches: [master] + branches: + - master jobs: pre-commit: diff --git a/.github/workflows/test_pytx.yml b/.github/workflows/test_pytx.yml index ffa2110..2814d27 100644 --- a/.github/workflows/test_pytx.yml +++ b/.github/workflows/test_pytx.yml @@ -2,7 +2,11 @@ name: Test Pytx on: pull_request: + branches: + - master push: + branches: + - master jobs: test: From ba090a746a45581e47754a39e1b59b3ee5eebc2b Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 09:56:25 +0100 Subject: [PATCH 26/34] Added __main__ --- pytransifex/__main__.py | 4 ++++ pytransifex/cli.py | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 pytransifex/__main__.py diff --git a/pytransifex/__main__.py b/pytransifex/__main__.py new file mode 100644 index 0000000..b6636bf --- /dev/null +++ b/pytransifex/__main__.py @@ -0,0 +1,4 @@ +from pytransifex.cli import cli + +if __name__ == "__main__": + cli() diff --git a/pytransifex/cli.py b/pytransifex/cli.py index d8db4e3..f9e45da 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -129,7 +129,3 @@ def pull(output_directory: str | Path | None, only_lang: str | None): finally: click.echo(reply) settings.to_disk() - - -if __name__ == "__main__": - cli() From aeb0c27f21694ade600321886af7f07dad53f954 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 10:10:26 +0100 Subject: [PATCH 27/34] Added runtime check+exception, explicit Python setup in CI, now requires 3.10 for type hints --- .github/workflows/test_pytx.yml | 4 ++++ pytransifex/__init__.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/test_pytx.yml b/.github/workflows/test_pytx.yml index 2814d27..f76e586 100644 --- a/.github/workflows/test_pytx.yml +++ b/.github/workflows/test_pytx.yml @@ -15,6 +15,10 @@ jobs: - name: Check out repository code uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Run CI tests env: organization: ${{ secrets.ORGANIZATION }} diff --git a/pytransifex/__init__.py b/pytransifex/__init__.py index ca0e42e..13e9447 100755 --- a/pytransifex/__init__.py +++ b/pytransifex/__init__.py @@ -1,3 +1,9 @@ import logging +from sys import version_info + +if version_info.major != 3 or version_info.minor < 10: + raise RuntimeError( + f"This program requires Python 3.10 at least, but found {version_info.major}.{version_info.minor}" + ) logging.basicConfig(level=logging.INFO) From 8a93c4ec41ac723c506d674f975da2b14c08a8a8 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 10:49:02 +0100 Subject: [PATCH 28/34] with open --- pytransifex/api.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pytransifex/api.py b/pytransifex/api.py index be09c1b..38b4532 100644 --- a/pytransifex/api.py +++ b/pytransifex/api.py @@ -116,8 +116,10 @@ def create_resource( with open(path_to_file, "r") as fh: content = fh.read() - tx_api.ResourceStringsAsyncUpload.upload(content, resource=resource) - logging.info(f"Resource created: {resource_slug or resource_name}") + + tx_api.ResourceStringsAsyncUpload.upload(content, resource=resource) + logging.info(f"Resource created: {resource_slug or resource_name}") + else: raise Exception( f"Not project could be found wiht the slug '{project_slug}'. Please create a project first." @@ -141,11 +143,10 @@ def update_source_translation( if resource := resources.get(slug=resource_slug): with open(path_to_file, "r") as fh: content = fh.read() - tx_api.ResourceStringsAsyncUpload.upload( - content, resource=resource - ) - logging.info(f"Source updated for resource: {resource_slug}") - return + + tx_api.ResourceStringsAsyncUpload.upload(content, resource=resource) + logging.info(f"Source updated for resource: {resource_slug}") + return raise Exception( f"Unable to find resource '{resource_slug}' in project '{project_slug}'" From 231f41fd46a77594b6091908f1156e8000fc916c Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 10:59:34 +0100 Subject: [PATCH 29/34] Ensuring paths for test inputs --- pytransifex/cli.py | 4 ++-- tests/test_api.py | 5 +++++ tests/test_cli.py | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pytransifex/cli.py b/pytransifex/cli.py index f9e45da..f7015b8 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -93,7 +93,7 @@ def push(input_directory: str | None): path_to_files=files, ) except Exception as error: - reply += f"Failed because of this error: {error}" + reply += f"cli:push failed because of this error: {error}" finally: click.echo(reply) settings.to_disk() @@ -125,7 +125,7 @@ def pull(output_directory: str | Path | None, only_lang: str | None): output_dir=output_directory, ) except Exception as error: - reply += f"Failed because of this error: {error}" + reply += f"cli:pull failed because of this error: {error}" finally: click.echo(reply) settings.to_disk() diff --git a/tests/test_api.py b/tests/test_api.py index e493ee6..25efa03 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -17,6 +17,11 @@ def setUpClass(cls): cls.path_to_file = Path.cwd().joinpath("tests", "input", "test_resource_fr.po") cls.output_dir = Path.cwd().joinpath("tests", "output") + if missing := next(filter(lambda p: not p.exists(), [cls.path_to_file]), None): + raise ValueError( + f"Unable to complete test with broken tests inputs. Found missing: {missing}" + ) + if project := cls.tx.get_project(project_slug=cls.project_slug): print("Found old project, removing.") project.delete() diff --git a/tests/test_cli.py b/tests/test_cli.py index 9a81a9c..dfb3bc1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -20,6 +20,14 @@ def setUpClass(cls): cls.path_to_file = cls.path_to_input_dir.joinpath("test_resource_fr.po") cls.output_dir = Path.cwd().joinpath("tests", "output") + if missing := next( + filter(lambda p: not p.exists(), [cls.path_to_file, cls.path_to_input_dir]), + None, + ): + raise ValueError( + f"Unable to complete test with broken tests inputs. Found missing: {missing}" + ) + if project := cls.tx.get_project(project_slug=cls.project_slug): print("Found old project, removing.") project.delete() From 634f59fefb8a67e48f107b3756dfdf53163d1bf9 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 11:13:56 +0100 Subject: [PATCH 30/34] Better error reporting --- pytransifex/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytransifex/cli.py b/pytransifex/cli.py index f7015b8..c16d9d8 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -19,7 +19,7 @@ def extract_files(input_dir: Path) -> tuple[list[Path], list[str], str]: files = list(Path.iterdir(input_dir)) slugs = path_to_slug(files) files_status_report = "\n".join( - (f"{slug} => {file}" for slug, file in zip(slugs, files)) + (f"file:{file} => slug:{slug}" for slug, file in zip(slugs, files)) ) return (files, slugs, files_status_report) From 99ed7d0075dfcb772e04d0b55c04ffbe468acccb Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 11:34:15 +0100 Subject: [PATCH 31/34] Pushing source files with no matching resource now creates the corresponding resource first. --- .github/workflows/test_pytx.yml | 2 +- pytransifex/api.py | 41 +++++++++++++++++++++++++++------ pytransifex/cli.py | 20 ++++++++++------ pytransifex/utils.py | 11 ++++++--- 4 files changed, 56 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test_pytx.yml b/.github/workflows/test_pytx.yml index f76e586..0ab8014 100644 --- a/.github/workflows/test_pytx.yml +++ b/.github/workflows/test_pytx.yml @@ -1,4 +1,4 @@ -name: Test Pytx +name: test pytransifex on: pull_request: diff --git a/pytransifex/api.py b/pytransifex/api.py index 38b4532..fd4ee90 100644 --- a/pytransifex/api.py +++ b/pytransifex/api.py @@ -138,6 +138,10 @@ def update_source_translation( "Unable to fetch resource for this organization; define an 'organization slug' first." ) + logging.info( + f"Updating source translation for resource {resource_slug} from file {path_to_file} (project: {project_slug})." + ) + if project := self.get_project(project_slug=project_slug): if resources := project.fetch("resources"): if resource := resources.get(slug=resource_slug): @@ -267,36 +271,59 @@ def pull( language_codes: list[str], output_dir: Path, ): + """Pull resources from project.""" args = [] for l_code in language_codes: for slug in resource_slugs: args.append(tuple([project_slug, slug, l_code, output_dir])) - logging.info("ARGS", args) - concurrently( + + res = concurrently( fn=self.get_translation, args=args, ) + logging.info(f"Pulled {args} for {len(res)} results).") + @ensure_login def push( self, project_slug: str, resource_slugs: list[str], path_to_files: list[Path] ): + """Push resources with files under project.""" if len(resource_slugs) != len(path_to_files): - raise Exception( + raise ValueError( f"Resources slugs ({len(resource_slugs)}) and Path to files ({len(path_to_files)}) must be equal in size!" ) + resource_zipped_with_path = list(zip(resource_slugs, path_to_files)) + resources = self.list_resources(project_slug) + logging.info( + f"Found {len(resources)} resource(s) for {project_slug}. Checking for missing resources and creating where necessary." + ) + created_when_missing_resource = [] + + for slug, path in resource_zipped_with_path: + logging.info(f"Slug: {slug}. Resources: {resources}.") + if not slug in resources: + logging.info( + f"{project_slug} is missing {slug}. Created it from {path}." + ) + self.create_resource( + project_slug=project_slug, path_to_file=path, resource_slug=slug + ) + created_when_missing_resource.append(slug) + args = [ - tuple([project_slug, res, path]) - for res, path in zip(resource_slugs, path_to_files) + tuple([project_slug, slug, path]) + for slug, path in resource_zipped_with_path + if not slug in created_when_missing_resource ] - concurrently( + res = concurrently( fn=self.update_source_translation, args=args, ) - logging.info(f"Pushes some {len(resource_slugs)} files!") + logging.info(f"Pushed {args} for {len(res)} results.") class Transifex: diff --git a/pytransifex/cli.py b/pytransifex/cli.py index c16d9d8..ca64746 100644 --- a/pytransifex/cli.py +++ b/pytransifex/cli.py @@ -1,3 +1,5 @@ +import logging +import traceback from os import mkdir, rmdir from pathlib import Path @@ -19,7 +21,7 @@ def extract_files(input_dir: Path) -> tuple[list[Path], list[str], str]: files = list(Path.iterdir(input_dir)) slugs = path_to_slug(files) files_status_report = "\n".join( - (f"file:{file} => slug:{slug}" for slug, file in zip(slugs, files)) + (f"file: {file} => slug: {slug}" for slug, file in zip(slugs, files)) ) return (files, slugs, files_status_report) @@ -54,7 +56,9 @@ def init(**opts): reply += f"WARNING: You will need to declare an input directory if you plan on using 'pytx push', as in 'pytx push --input-directory '." except Exception as error: - reply += f"Failed to initialize the CLI, this error occurred: {error} " + reply += ( + f"cli:init > Failed to initialize the CLI, this error occurred: {error}." + ) if has_to_create_dir: rmdir(settings.output_directory) @@ -78,14 +82,14 @@ def push(input_directory: str | None): ) if not input_dir: - raise Exception( - "To use this 'push', you need to initialize the project with a valid path to the directory containing the files to push; alternatively, you can call this commend with 'pytx push --input-directory '." + raise FileExistsError( + "cli:push > To use this 'push', you need to initialize the project with a valid path to the directory containing the files to push; alternatively, you can call this commend with 'pytx push --input-directory '." ) try: files, slugs, files_status_report = extract_files(input_dir) click.echo( - f"Pushing {files_status_report} to Transifex under project {settings.project_slug}..." + f"cli:push > Pushing {files_status_report} to Transifex under project {settings.project_slug}." ) client.push( project_slug=settings.project_slug, @@ -93,7 +97,8 @@ def push(input_directory: str | None): path_to_files=files, ) except Exception as error: - reply += f"cli:push failed because of this error: {error}" + reply += f"cli:push > Failed because of this error: {error}" + logging.error(f"traceback: {traceback.print_exc()}") finally: click.echo(reply) settings.to_disk() @@ -125,7 +130,8 @@ def pull(output_directory: str | Path | None, only_lang: str | None): output_dir=output_directory, ) except Exception as error: - reply += f"cli:pull failed because of this error: {error}" + reply += f"cli:pull > failed because of this error: {error}" + logging.error(f"traceback: {traceback.print_exc()}") finally: click.echo(reply) settings.to_disk() diff --git a/pytransifex/utils.py b/pytransifex/utils.py index 26e894c..f125295 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -1,6 +1,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from functools import wraps -from typing import Any, Callable, Iterable +from typing import Any, Callable def ensure_login(f): @@ -16,9 +16,14 @@ def capture_args(instance, *args, **kwargs): def concurrently( *, fn: Callable | None = None, - args: Iterable[Any] | None = None, - partials: Iterable[Any] | None = None, + args: list[Any] | None = None, + partials: list[Any] | None = None, ) -> list[Any]: + if args and len(args) == 0: + return [] + if partials and len(partials) == 0: + return [] + with ThreadPoolExecutor() as pool: if partials: futures = [pool.submit(p) for p in partials] From 06c031d1ad28500842210cc50337b0267c5b410b Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Wed, 15 Feb 2023 14:38:00 +0100 Subject: [PATCH 32/34] Fixing incorrect 'list_resources' method. --- pytransifex/api.py | 18 ++++++++++-------- pytransifex/utils.py | 14 +++++++------- tests/test_cli.py | 2 +- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/pytransifex/api.py b/pytransifex/api.py index fd4ee90..ebf2798 100644 --- a/pytransifex/api.py +++ b/pytransifex/api.py @@ -84,12 +84,14 @@ def get_project(self, project_slug: str) -> None | Resource: return None @ensure_login - def list_resources(self, project_slug: str) -> list[Resource]: + def list_resources(self, project_slug: str) -> list[Any]: """List all resources for the project passed as argument""" - if self.projects: - res = self.projects.filter(slug=project_slug) - logging.info("Obtained these resources:") - return res + if project := self.get_project(project_slug=project_slug): + if resources := project.fetch("resources"): + return list(resources.all()) + else: + return [] + raise Exception( f"Unable to find any project under this organization: '{self.organization}'" ) @@ -109,8 +111,8 @@ def create_resource( if project := self.get_project(project_slug=project_slug): resource = tx_api.Resource.create( project=project, - name=resource_name, - slug=resource_slug, + name=resource_name or resource_slug, + slug=resource_slug or resource_name, i18n_format=tx_api.I18nFormat(id=self.i18n_type), ) @@ -305,7 +307,7 @@ def push( logging.info(f"Slug: {slug}. Resources: {resources}.") if not slug in resources: logging.info( - f"{project_slug} is missing {slug}. Created it from {path}." + f"{project_slug} is missing {slug}. Creating it from {path}." ) self.create_resource( project_slug=project_slug, path_to_file=path, resource_slug=slug diff --git a/pytransifex/utils.py b/pytransifex/utils.py index f125295..ea28b14 100644 --- a/pytransifex/utils.py +++ b/pytransifex/utils.py @@ -19,16 +19,16 @@ def concurrently( args: list[Any] | None = None, partials: list[Any] | None = None, ) -> list[Any]: - if args and len(args) == 0: - return [] - if partials and len(partials) == 0: - return [] with ThreadPoolExecutor() as pool: - if partials: + if not partials is None: + assert args is None and fn is None futures = [pool.submit(p) for p in partials] - elif fn and args: + elif (not args is None) and (not fn is None): + assert partials is None futures = [pool.submit(fn, *a) for a in args] else: - raise Exception("Either partials or fn and args!") + raise ValueError( + "Exactly 1 of 'partials' or 'args' must be defined. Found neither was when calling concurrently." + ) return [f.result() for f in as_completed(futures)] diff --git a/tests/test_cli.py b/tests/test_cli.py index dfb3bc1..3b8af21 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -54,7 +54,7 @@ def test1_init(self): def test2_push(self): result = self.runner.invoke(cli, ["push", "-in", str(self.path_to_input_dir)]) - passed = result.exit_code == 0 and "No such file" in result.output + passed = result.exit_code == 0 print(result.output) assert passed From b2d7917fe84be28d9102b47aa57da89f4363f920 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 16 Feb 2023 08:57:49 +0100 Subject: [PATCH 33/34] Added return value for 'ping', as I remember it's used in assertions by users --- pytransifex/api.py | 3 ++- pytransifex/interfaces.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pytransifex/api.py b/pytransifex/api.py index ebf2798..0f59eaa 100644 --- a/pytransifex/api.py +++ b/pytransifex/api.py @@ -249,12 +249,13 @@ def project_exists(self, project_slug: str) -> bool: ) @ensure_login - def ping(self): + def ping(self) -> bool: """ Exposing this just for the sake of satisfying qgis-plugin-cli's expectations There is no need to ping the server on the current implementation, as connection is handled by the SDK """ logging.info("'ping' is deprecated!") + return True @ensure_login def get_project_stats(self, project_slug: str) -> dict[str, Any]: diff --git a/pytransifex/interfaces.py b/pytransifex/interfaces.py index 562ebf9..8268dd3 100644 --- a/pytransifex/interfaces.py +++ b/pytransifex/interfaces.py @@ -6,8 +6,7 @@ class Tx(ABC): # TODO # This interface modelled after api.py:Transifex satisfies the expectations of qgis-plugin-cli # but it falls short of requiring an implementation for methods that qgis-plugin-cli does not use: - # coordinator, create_translation, delete_project, delete_resource, delete_team - # The question is whether I should implement them. Is there's a consumer downstream? + # { coordinator, create_translation, delete_project, delete_resource, delete_team } @abstractmethod def create_project( @@ -77,5 +76,5 @@ def project_exists(self, project_slug: str) -> bool: raise NotImplementedError @abstractmethod - def ping(self): + def ping(self) -> bool: raise NotImplementedError From 8b8046bc9fd092def492d126c6b1a2104f47f983 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Thu, 16 Feb 2023 09:04:56 +0100 Subject: [PATCH 34/34] Removed test for 3.10 from __init__ --- pytransifex/__init__.py | 6 ------ setup.py | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pytransifex/__init__.py b/pytransifex/__init__.py index 13e9447..ca0e42e 100755 --- a/pytransifex/__init__.py +++ b/pytransifex/__init__.py @@ -1,9 +1,3 @@ import logging -from sys import version_info - -if version_info.major != 3 or version_info.minor < 10: - raise RuntimeError( - f"This program requires Python 3.10 at least, but found {version_info.major}.{version_info.minor}" - ) logging.basicConfig(level=logging.INFO) diff --git a/setup.py b/setup.py index a867238..bac75ad 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup -python_min_version = (3, 8) +python_min_version = (3, 10) if sys.version_info < python_min_version: sys.exit(