From 2e715f01821c2fb4989ff4a57761b625884a363d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Maillard?= Date: Thu, 19 Oct 2017 21:24:32 +0800 Subject: [PATCH 1/5] Refactor auto addons to fix addons_path for Odoo standard addons --- auto_addons/addons.py | 340 +++++++++++++++++++++--------------------- 1 file changed, 173 insertions(+), 167 deletions(-) diff --git a/auto_addons/addons.py b/auto_addons/addons.py index cbf8f31d..20547e72 100644 --- a/auto_addons/addons.py +++ b/auto_addons/addons.py @@ -3,22 +3,18 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). import os import re +import shutil +import subprocess import sys -from os import remove -from shutil import move -from urlparse import urlparse -from subprocess import call -from subprocess import check_output +import urlparse EXTRA_ADDONS_PATH = '/opt/odoo/additional_addons/' -OLD_ODOO_CONF = '/opt/odoo/etc/odoo.conf.old' +ODOO_ADDONS_PATH = '/opt/odoo/sources/odoo/addons' ODOO_CONF = '/opt/odoo/etc/odoo.conf' -ADDONS_PATH = ['/opt/odoo/sources/odoo/addons'] -DEFAULT_SCHEME = 'https://' -DEFAULT_GIT_REPO_HOSTING_SERVICE = 'github.com' +DEFAULT_SCHEME = 'https' +DEFAULT_GIT_HOSTING_SERVICE = 'github.com' DEFAULT_ORGANIZATION = 'OCA' DEPENDENCIES_FILE = 'oca_dependencies.txt' -REGEX_ADDONS_PATH = r'^addons_path\s*=\s*' class Repo(object): @@ -31,124 +27,109 @@ class Repo(object): Following the oca_dependencies.txt syntax: https://github.com/OCA/maintainer-quality-tools/blob/master/sample_files/oca_dependencies.txt """ - def __init__(self, remote_url, parent=None): - if parent: - self.parent = parent - self.branch = self.parent.branch - else: - self.parent = None - self.branch = None + def __init__(self, remote_url, fetch_dep, parent=None): self.remote_url = remote_url - self.folder_name = None + self.fetch_dep = fetch_dep + self.parent = parent + + self.scheme = DEFAULT_SCHEME + self.netloc = DEFAULT_GIT_HOSTING_SERVICE self.organization = DEFAULT_ORGANIZATION self.repository = None - self.scheme = DEFAULT_SCHEME - self.git_repo_host = DEFAULT_GIT_REPO_HOSTING_SERVICE + self.branch = self.parent.branch if parent else None + self.folder = None + self._parse() - def _set_branch(self, branch): - self.branch = branch if branch else self.parent.branch - - def _check_is_ssh(self, url): - # TODO For other hosting services, this part should be dynamic. - # TODO support URL like ssh://git@gitlab.domain.name:10022 - # TODO Maybe we could consider using a standard URL parser in the future. - if url.startswith('git@github.com:'): - self.scheme = 'git@' - self.git_repo_host = 'github.com:' - return True - - def _check_is_url(self, url): - return re.match(r'^https?:/{2}\w.+$', url) \ - or self._check_is_ssh(url) - - def fetch_branch_name(self): - branch_cmd = 'git --git-dir=%s/.git --work-tree=%s branch' % ( - self.path, self.path - ) - output = check_output(branch_cmd, shell=True) - for line in output.split('\n'): - if line.startswith('*'): - self._set_branch(line.replace('* ', '')) - - def _parse_organization_repo(self, remote_url): - _args = remote_url.split('/') - _len_args = len(_args) - if _len_args == 1: - # repo - self.repository = _args[0] - self.folder_name = self.repository - elif _len_args == 2: - # organization AND repo - self.organization = _args[0] - self.repository = _args[1] - self.folder_name = self.repository + @staticmethod + def _is_http(url): + # FIXME should also support following syntax (different from Git SSH): + # 'ssh://git@github.com/organization/private-repo' + return re.match(r'^https?:/{2}.+', url) + + @staticmethod + def _is_git_ssh(url): + return re.match(r'^\w+@\w+\.\w+:.+', url) + + @staticmethod + def _is_url(url): + return _is_http(url) or _is_git_ssh(url) + + def _parse_org_repo(self, repo): + args = repo.split('/') + if len(args) == 1: + # Pattern: 'public-repo' + self.repository = args[0] + elif len(args) == 2: + # Pattern: 'organization/public-repo' + self.organization = args[0] + self.repository = args[1] def _parse_url(self, url): - if self.scheme == DEFAULT_SCHEME: - _url_parse = urlparse(url) - self.git_repo_host = _url_parse.netloc - _args_path = _url_parse.path.split('/') - self.organization = _args_path[1] - self.repository = _args_path[2].replace('.git', '') - self.folder_name = self.repository - elif self.scheme == 'git@': - _args = url.split(':')[1] - _args_path = _args.split('/') - self.organization = _args_path[0] - self.repository = _args_path[1].replace('.git', '') - self.folder_name = self.repository + path = None + if self._is_http(url): + # Pattern: 'https://github.com/organization/public-repo' + parsed_url = urlparse.urlparse(url) + self.scheme = parsed_url.scheme + self.netloc = parsed_url.netloc + path = parsed_url.path + else: + # Pattern: 'git@github.com:organization/private-repo' + self.scheme = 'ssh' + args = url.split(':') + self.netloc = args[0] + path = args[1] + args = path.split('/') + self.organization = args[1] + # Repo might end with '.git' but it's optional + self.repository = args[2].replace('.git', '') + + def _parse_repo(self, repo): + if self._is_url(args[1]): + self._parse_url(repo) + else + self._parse_org_repo(repo) def _parse(self): - _remote_url = self.remote_url - _args = _remote_url.split(' ') - _len_args = len(_args) - if _len_args == 1: - if self._check_is_url(_args[0]): - # url - self._parse_url(_args[0]) + # Clean repo (remove multiple/trailing spaces) + repo = re.sub('\s+', ' ', self.remote_url.strip()) + + # Check if folder and/or branch are provided + args = repo.split(' ') + + if len(args) == 1: + # Pattern: 'repo' + _parse_repo(repo) + self.folder = self.repository + if len(args) == 2: + if self._is_url(args[1]): + # Pattern: 'folder repo' + # This pattern is only valid if the repo is a URL + self._parse_url(args[1]) + self.folder = args[0] else: - # repo OR organization/repo - self._parse_organization_repo(_args[0]) - elif _len_args == 2: - if self._check_is_url(_args[0]): - # url AND branch - self._parse_url(_args[0]) - self._set_branch(_args[1]) - else: - if self._check_is_url(_args[1]): - # repo AND url - self._parse_url(_args[1]) - self.folder_name = _args[0] - else: - # repo OR organization/repo AND branch - self._parse_organization_repo(_args[0]) - self._set_branch(_args[1]) - elif _len_args == 3: - if self._check_is_url(_args[1]): - # repo OR organization/repo AND url AND branch - self._parse_organization_repo(_args[0]) - self._parse_url(_args[1]) - self._set_branch(_args[2]) - self.folder_name = _args[0] + # Pattern: 'repo branch' + self._parse_repo(args[0]) + self.folder = self.repository + self.branch = args[1] + elif len(args) == 3: + # Pattern: 'folder repo branch' + self._parse_repo(args[1]) + self.folder = args[0] + self.branch = args[2] @property def path(self): - return '%s%s' % (EXTRA_ADDONS_PATH, self.folder_name) + return '%s%s' % (EXTRA_ADDONS_PATH, self.folder) @property def resolve_url(self): - _resolve_url_str = '%s%s/%s/%s.git' - if self.scheme == 'git@': - _resolve_url_str = '%s%s%s/%s.git' - - _resolve_url = _resolve_url_str % ( + return '%s://%s/%s/%s.git' % ( self.scheme, - self.git_repo_host, + self.netloc, self.organization, self.repository ) - return _resolve_url @property def download_cmd(self): @@ -164,81 +145,101 @@ def download_cmd(self): @property def update_cmd(self): - if self.branch: - cmd = 'git --git-dir=%s/.git --work-tree=%s pull origin %s' % ( - self.path, self.path, self.branch - ) - return cmd.split() - else: - self.fetch_branch_name() - cmd = 'git --git-dir=%s/.git --work-tree=%s pull origin %s' % ( - self.path, self.path, self.branch - ) - return cmd.split() + cmd = 'git -C %s pull origin %s' % ( + self.path, self.branch + ) + return cmd.split() + + def _fetch_branch_name(self): + # Example of output from `git branch` command: + # 7.0 + # 7.0.1.0 + # * 7.0.1.1 + # 8.0 + # 8.0.1.0 + # 9.0 + # 9.0.1.0 + branch_cmd = 'git -C %s branch' % ( + self.path + ) - def download(self, parent=None, is_loop=False, fetch_dep=True): - if self.path in ADDONS_PATH: + # Search for the branch prefixed with '* ' + output = subprocess.check_output(branch_cmd, shell=True) + for line in output.split('\n'): + if line.startswith('*'): + self.branch = line.replace('* ', '') + + def _download_dependencies(self, addons_path): + # Check if the repo contains a dependency file + filename = '%s/%s' % (self.path, DEPENDENCIES_FILE) + if not os.path.exists(filename): return + + # Download the dependencies + with open(filename) as f: + for line in f: + l = line.strip('\n').strip() + if l and not l.startswith('#'): + Repo(l, self).download(addons_path) + + def download(self, addons_path, parent=None, is_retry=False): + # No need to fetch a repo twice (it could also cause infinite loop) + if self.path in addons_path: + return + if os.path.exists(self.path): - if fetch_dep: - cmd = self.update_cmd - call(cmd) - else: - self.fetch_branch_name() + # Branch name is used to: + # - pull the code + # - fetch the child repos + self._fetch_branch_name() + + if self.fetch_dep: + # Perform `git pull` + subprocess.call(self.update_cmd) else: - result = call(self.download_cmd) + # Perform `git clone` + result = subprocess.call(self.download_cmd) + if result != 0: + # Since `git clone` failed, try some workarounds if parent and parent.parent: + # Retry recursively using the ancestors' branch self.branch = parent.parent.branch self.download( parent=parent.parent, - is_loop=True, - fetch_dep=fetch_dep) + is_retry=True) else: + # Retry with the default branch of the repo self.branch = None - self.download( - is_loop=True, - fetch_dep=fetch_dep) + self.download(is_retry=True) else: - self.fetch_branch_name() + # Branch name is used to fetch the child repos + self._fetch_branch_name() - if not is_loop: - ADDONS_PATH.append(self.path) - self.download_dependency(fetch_dep) + if not is_retry: + addons_path.append(self.path) + self._download_dependencies(addons_path) - def download_dependency(self, fetch_dep=True): - filename = '%s/%s' % (self.path, DEPENDENCIES_FILE) - if not os.path.exists(filename): - return - repo_list = [] - with open(filename) as f: - for line in f: - l = line.strip('\n').strip() - if l.startswith('#') or not l: - continue - repo_list.append(Repo(l, self)) - for repo in repo_list: - repo.download( - parent=repo.parent, - fetch_dep=fetch_dep) +def write_addons_path(addons_path): + conf_file = ODOO_CONF + '.new' -def write_addons_path(): - move(ODOO_CONF, OLD_ODOO_CONF) - with open(ODOO_CONF, 'a') as target_file, open(OLD_ODOO_CONF, 'r') as source_file: - for line in source_file: - if not re.match(REGEX_ADDONS_PATH, line): - target_file.write(line) + with open(conf_file, 'a') as target, open(ODOO_CONF, 'r') as source: + # Copy all lines except for the addons_path parameter + for line in source: + if not re.match(r'^addons_path\s*=\s*', line): + target.write(line) - new_line = 'addons_path = %s' % ','.join(list(set(ADDONS_PATH))) - target_file.write(new_line) + # Append addons_path + target_file.write('addons_path = %s' % ','.join(addons_path)) - remove(OLD_ODOO_CONF) + shutil.move(conf_file, ODOO_CONF) def main(): fetch_dep = True remote_url = None + addons_path = [] # 1st param is FETCH_OCA_DEPENDENCIES if len(sys.argv) > 1: @@ -251,23 +252,28 @@ def main(): # If the ADDONS_REPO contains a branch name, there is a space before the # branch name so the branch name becomes the 3rd param + # FIXME TBC if it's still the case since bash variables are protected by "" if len(sys.argv) > 3: remote_url += ' ' + sys.argv[3] if remote_url: # Only one master repo to download - Repo(remote_url).download(fetch_dep=fetch_dep) + Repo(remote_url, fetch_dep).download(addons_path) else: - # List of repos is defined in oca_dependencies.txt at the root of - # additional_addons folder, let's download them all + # List of repos defined in oca_dependencies.txt at the root of + # additional_addons folder, download them all with open(EXTRA_ADDONS_PATH + DEPENDENCIES_FILE, 'r') as f: for line in f: l = line.strip('\n').strip() - if l.startswith('#') or not l: - continue - Repo(l).download(fetch_dep=fetch_dep) + if l and not l.startswith('#'): + Repo(l, fetch_dep).download(addons_path) - write_addons_path() + # Odoo standard addons path must be the last, in case an additional addon + # uses the same name as a standard module (e.g. Odoo EE 'web' module in v9) + addons_path.append(ODOO_ADDONS_PATH) + + write_addons_path(addons_path) if __name__ == '__main__': main() + From a3efaacd6fc910a460e3269024aeee8eddce5a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Maillard?= Date: Fri, 20 Oct 2017 10:25:00 +0800 Subject: [PATCH 2/5] network_bridge is useless --- README.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/README.md b/README.md index 9106eb83..c185cd5c 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,6 @@ more information about Compose, see the [official documentation][dc-doc]. image: postgres:9.5 environment: - POSTGRES_USER=odoo - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -93,7 +92,6 @@ more information about Compose, see the [official documentation][dc-doc]. - postgres:db environment: - ODOO_DB_USER=odoo - network_mode: bridge Once this file is created, simply move to the corresponding folder and run the following command to start Odoo: @@ -150,7 +148,6 @@ The `docker-compose.yml` should look like: environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=strong_pg_superuser_password - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -163,7 +160,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** If Odoo is behind a reverse proxy, it is also suggested to change the port published by the container (though this port is actually not opened to the @@ -205,7 +201,6 @@ The `docker-compose.yml` should look like: environment: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=strong_pg_superuser_password - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -221,7 +216,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** With this configuration, all the data created in the volumes will belong to the user whose UID matches the user running inside the container. @@ -341,7 +335,6 @@ The `docker-compose.yml` should look like: - POSTGRES_PASSWORD=strong_pg_superuser_password - /etc/passwd:/etc/passwd:ro user: 1001:1001 - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -358,7 +351,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** For a more dynamic UID mapping, you can use Compose [variable substitution][dk-var]. Simply export the environment variable `UID` @@ -458,7 +450,6 @@ as well as all the Git repositories it depends on, you can use the following - POSTGRES_PASSWORD=strong_pg_superuser_password - /etc/passwd:/etc/passwd:ro user: 1001:1001 - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -477,7 +468,6 @@ as well as all the Git repositories it depends on, you can use the following - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** After the repositories have been fetched, it might not be required to pull them every time the container is restarted. In that case, simply set the @@ -532,7 +522,6 @@ The `docker-compose.yml` should look like: - POSTGRES_PASSWORD=strong_pg_superuser_password - /etc/passwd:/etc/passwd:ro user: 1001:1001 - network_mode: bridge odoo: image: elicocorp/odoo:10.0 @@ -552,7 +541,6 @@ The `docker-compose.yml` should look like: - ODOO_ADMIN_PASSWD=strong_odoo_master_password - ODOO_DB_USER=odoo - ODOO_DB_PASSWORD=strong_pg_odoo_password - network_mode: bridge **Note:** If the host user has a valid SSH key under the `.ssh` folder of his home folder, he can map his `.ssh` folder instead, e.g.: From 8e042d3c1c2c806aea298cb81d3843b94f7de198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Maillard?= Date: Fri, 20 Oct 2017 12:12:54 +0800 Subject: [PATCH 3/5] Fix syntax errors and add error log for malformed repos Error management allows to avoid infinite loops/recursivity. Using Exceptions would probably make it easier. --- auto_addons/addons.py | 93 +++++++++++++++++++++++++++++++------------ 1 file changed, 68 insertions(+), 25 deletions(-) diff --git a/auto_addons/addons.py b/auto_addons/addons.py index 20547e72..c34db4d5 100644 --- a/auto_addons/addons.py +++ b/auto_addons/addons.py @@ -36,7 +36,7 @@ def __init__(self, remote_url, fetch_dep, parent=None): self.netloc = DEFAULT_GIT_HOSTING_SERVICE self.organization = DEFAULT_ORGANIZATION self.repository = None - self.branch = self.parent.branch if parent else None + self.branch = parent.branch if parent else None self.folder = None self._parse() @@ -53,10 +53,11 @@ def _is_git_ssh(url): @staticmethod def _is_url(url): - return _is_http(url) or _is_git_ssh(url) + return Repo._is_http(url) or Repo._is_git_ssh(url) def _parse_org_repo(self, repo): args = repo.split('/') + if len(args) == 1: # Pattern: 'public-repo' self.repository = args[0] @@ -64,30 +65,59 @@ def _parse_org_repo(self, repo): # Pattern: 'organization/public-repo' self.organization = args[0] self.repository = args[1] + else: + print 'FATAL:' + print 'Expected pattern option #1: public-repo' + print 'Expected pattern option #2: organization/public-repo' + print 'Actual value: %s' % repo def _parse_url(self, url): path = None - if self._is_http(url): + + if Repo._is_http(url): # Pattern: 'https://github.com/organization/public-repo' parsed_url = urlparse.urlparse(url) + self.scheme = parsed_url.scheme self.netloc = parsed_url.netloc - path = parsed_url.path + + if len(parsed_url.path) > 0: + path = parsed_url.path[1:] + else: + print 'FATAL:' + print 'Expected pattern: https://github.com/organization/public-repo' + print 'Actual value: %s' % url + return else: # Pattern: 'git@github.com:organization/private-repo' self.scheme = 'ssh' args = url.split(':') - self.netloc = args[0] - path = args[1] + + if len(args) == 2: + self.netloc = args[0] + path = args[1] + else: + print 'FATAL:' + print 'Expected pattern: git@github.com:organization/private-repo' + print 'Actual value: %s' % url + return + + # Pattern: 'organization/repo' args = path.split('/') - self.organization = args[1] - # Repo might end with '.git' but it's optional - self.repository = args[2].replace('.git', '') + + if len(args) == 2: + self.organization = args[0] + # Repo might end with '.git' but it's optional + self.repository = args[1].replace('.git', '') + else: + print 'FATAL:' + print 'Expected pattern: organization/repo' + print 'Actual value: %s' % path def _parse_repo(self, repo): - if self._is_url(args[1]): + if Repo._is_url(repo): self._parse_url(repo) - else + else: self._parse_org_repo(repo) def _parse(self): @@ -99,10 +129,10 @@ def _parse(self): if len(args) == 1: # Pattern: 'repo' - _parse_repo(repo) + self._parse_repo(repo) self.folder = self.repository if len(args) == 2: - if self._is_url(args[1]): + if Repo._is_url(args[1]): # Pattern: 'folder repo' # This pattern is only valid if the repo is a URL self._parse_url(args[1]) @@ -165,9 +195,17 @@ def _fetch_branch_name(self): # Search for the branch prefixed with '* ' output = subprocess.check_output(branch_cmd, shell=True) + found = False for line in output.split('\n'): if line.startswith('*'): self.branch = line.replace('* ', '') + found = True + break + + if not found: + print 'FATAL:' + print 'Cannot fetch branch name' + print 'Path: %s' % self.path def _download_dependencies(self, addons_path): # Check if the repo contains a dependency file @@ -180,9 +218,10 @@ def _download_dependencies(self, addons_path): for line in f: l = line.strip('\n').strip() if l and not l.startswith('#'): - Repo(l, self).download(addons_path) + Repo(l, self.fetch_dep, self).download(addons_path) def download(self, addons_path, parent=None, is_retry=False): + # No need to fetch a repo twice (it could also cause infinite loop) if self.path in addons_path: return @@ -195,7 +234,12 @@ def download(self, addons_path, parent=None, is_retry=False): if self.fetch_dep: # Perform `git pull` - subprocess.call(self.update_cmd) + result = subprocess.call(self.update_cmd) + + if result != 0: + print 'FATAL:' + print 'Cannot git pull repository' + print 'URL: %s' % self.remote_url else: # Perform `git clone` result = subprocess.call(self.download_cmd) @@ -206,12 +250,17 @@ def download(self, addons_path, parent=None, is_retry=False): # Retry recursively using the ancestors' branch self.branch = parent.parent.branch self.download( + addons_path, parent=parent.parent, is_retry=True) - else: - # Retry with the default branch of the repo + elif not is_retry: + # Retry one last time with the default branch of the repo self.branch = None - self.download(is_retry=True) + self.download(addons_path, is_retry=True) + else: + print 'FATAL:' + print 'Cannot git clone repository' + print 'URL: %s' % self.remote_url else: # Branch name is used to fetch the child repos self._fetch_branch_name() @@ -231,7 +280,7 @@ def write_addons_path(addons_path): target.write(line) # Append addons_path - target_file.write('addons_path = %s' % ','.join(addons_path)) + target.write('addons_path = %s' % ','.join(addons_path)) shutil.move(conf_file, ODOO_CONF) @@ -250,12 +299,6 @@ def main(): if len(sys.argv) > 2: remote_url = sys.argv[2] - # If the ADDONS_REPO contains a branch name, there is a space before the - # branch name so the branch name becomes the 3rd param - # FIXME TBC if it's still the case since bash variables are protected by "" - if len(sys.argv) > 3: - remote_url += ' ' + sys.argv[3] - if remote_url: # Only one master repo to download Repo(remote_url, fetch_dep).download(addons_path) From 7bc1028170eeae7bc367980ebddfa117e9ef1dbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Maillard?= Date: Fri, 20 Oct 2017 12:49:15 +0800 Subject: [PATCH 4/5] Manage error when repository was not pulled correctly Also print to STDERR to flush automatically so that error messages appear instantly --- auto_addons/addons.py | 64 ++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/auto_addons/addons.py b/auto_addons/addons.py index c34db4d5..b042be77 100644 --- a/auto_addons/addons.py +++ b/auto_addons/addons.py @@ -66,10 +66,10 @@ def _parse_org_repo(self, repo): self.organization = args[0] self.repository = args[1] else: - print 'FATAL:' - print 'Expected pattern option #1: public-repo' - print 'Expected pattern option #2: organization/public-repo' - print 'Actual value: %s' % repo + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern #1: public-repo' + print >> sys.stderr, 'Expected pattern #2: organization/public-repo' + print >> sys.stderr, 'Actual value: %s' % repo def _parse_url(self, url): path = None @@ -84,9 +84,9 @@ def _parse_url(self, url): if len(parsed_url.path) > 0: path = parsed_url.path[1:] else: - print 'FATAL:' - print 'Expected pattern: https://github.com/organization/public-repo' - print 'Actual value: %s' % url + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern: https://github.com/organization/public-repo' + print >> sys.stderr, 'Actual value: %s' % url return else: # Pattern: 'git@github.com:organization/private-repo' @@ -97,9 +97,9 @@ def _parse_url(self, url): self.netloc = args[0] path = args[1] else: - print 'FATAL:' - print 'Expected pattern: git@github.com:organization/private-repo' - print 'Actual value: %s' % url + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern: git@github.com:organization/private-repo' + print >> sys.stderr, 'Actual value: %s' % url return # Pattern: 'organization/repo' @@ -110,9 +110,9 @@ def _parse_url(self, url): # Repo might end with '.git' but it's optional self.repository = args[1].replace('.git', '') else: - print 'FATAL:' - print 'Expected pattern: organization/repo' - print 'Actual value: %s' % path + print >> sys.stderr, 'FATAL: unexpected repository pattern' + print >> sys.stderr, 'Expected pattern: organization/repo' + print >> sys.stderr, 'Actual value: %s' % path def _parse_repo(self, repo): if Repo._is_url(repo): @@ -194,18 +194,22 @@ def _fetch_branch_name(self): ) # Search for the branch prefixed with '* ' - output = subprocess.check_output(branch_cmd, shell=True) - found = False - for line in output.split('\n'): - if line.startswith('*'): - self.branch = line.replace('* ', '') - found = True - break - - if not found: - print 'FATAL:' - print 'Cannot fetch branch name' - print 'Path: %s' % self.path + try: + found = False + output = subprocess.check_output(branch_cmd, shell=True) + for line in output.split('\n'): + if line.startswith('*'): + self.branch = line.replace('* ', '') + found = True + break + + if not found: + print >> sys.stderr, 'FATAL: cannot fetch branch name' + print >> sys.stderr, 'Path: %s' % self.path + + except Exception, e: + print >> sys.stderr, 'FATAL: cannot fetch branch name' + print >> sys.stderr, e def _download_dependencies(self, addons_path): # Check if the repo contains a dependency file @@ -237,9 +241,8 @@ def download(self, addons_path, parent=None, is_retry=False): result = subprocess.call(self.update_cmd) if result != 0: - print 'FATAL:' - print 'Cannot git pull repository' - print 'URL: %s' % self.remote_url + print >> sys.stderr, 'FATAL: cannot git pull repository' + print >> sys.stderr, 'URL: %s' % self.remote_url else: # Perform `git clone` result = subprocess.call(self.download_cmd) @@ -258,9 +261,8 @@ def download(self, addons_path, parent=None, is_retry=False): self.branch = None self.download(addons_path, is_retry=True) else: - print 'FATAL:' - print 'Cannot git clone repository' - print 'URL: %s' % self.remote_url + print >> sys.stderr, 'FATAL: cannot git clone repository' + print >> sys.stderr, 'URL: %s' % self.remote_url else: # Branch name is used to fetch the child repos self._fetch_branch_name() From a773b22788c6c999a5ccd9304d17574ca6e36711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Maillard?= Date: Fri, 20 Oct 2017 13:08:44 +0800 Subject: [PATCH 5/5] Add log for git clone/pull --- auto_addons/addons.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/auto_addons/addons.py b/auto_addons/addons.py index b042be77..b58bbf38 100644 --- a/auto_addons/addons.py +++ b/auto_addons/addons.py @@ -131,6 +131,7 @@ def _parse(self): # Pattern: 'repo' self._parse_repo(repo) self.folder = self.repository + if len(args) == 2: if Repo._is_url(args[1]): # Pattern: 'folder repo' @@ -142,6 +143,7 @@ def _parse(self): self._parse_repo(args[0]) self.folder = self.repository self.branch = args[1] + elif len(args) == 3: # Pattern: 'folder repo branch' self._parse_repo(args[1]) @@ -171,14 +173,14 @@ def download_cmd(self): cmd = 'git clone %s %s' % ( self.resolve_url, self.path ) - return cmd.split() + return cmd @property def update_cmd(self): cmd = 'git -C %s pull origin %s' % ( self.path, self.branch ) - return cmd.split() + return cmd def _fetch_branch_name(self): # Example of output from `git branch` command: @@ -238,14 +240,18 @@ def download(self, addons_path, parent=None, is_retry=False): if self.fetch_dep: # Perform `git pull` - result = subprocess.call(self.update_cmd) + print 'Pulling: %s' % self.remote_url + sys.stdout.flush() + result = subprocess.call(self.update_cmd.split()) if result != 0: print >> sys.stderr, 'FATAL: cannot git pull repository' print >> sys.stderr, 'URL: %s' % self.remote_url else: # Perform `git clone` - result = subprocess.call(self.download_cmd) + print 'Cloning: %s' % self.remote_url + sys.stdout.flush() + result = subprocess.call(self.download_cmd.split()) if result != 0: # Since `git clone` failed, try some workarounds