Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

common: git retry feature #44

Merged
merged 1 commit into from
Oct 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 52 additions & 39 deletions repo_resource/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,16 @@
from typing import NamedTuple
from urllib.parse import urlparse
from multiprocessing import Pool
from retrying import retry

import ssh_agent_setup
from repo import manifest_xml
from repo import main as repo


DEFAULT_CHECK_JOBS = 2
GIT_LS_REMOTE_MAX_RETRIES = 3
GIT_LS_REMOTE_WAIT = 2000 # 2s
CACHEDIR = Path('/tmp/repo-resource-cache')
SHA1_PATTERN = re.compile(r'^[0-9a-f]{40}$')
EXCLUDE_ATTRS = {'dest-branch', 'upstream'}
Expand Down Expand Up @@ -88,51 +91,61 @@ def multi_run_wrapper(args):
return getRevision(*args)


def retry_getRevision(exception):
"""Return True if we should retry, raise exception otherwise"""
with redirect_stdout(sys.stderr):
print(exception)
if "HTTP 429" not in str(exception).upper():
raise str(exception)
print('git ls-remote: sleeping {:.1f} seconds before retrying'.format(
GIT_LS_REMOTE_WAIT // 1000)
)
return True


@retry(retry_on_exception=retry_getRevision,
stop_max_attempt_number=GIT_LS_REMOTE_MAX_RETRIES,
wait_fixed=GIT_LS_REMOTE_WAIT)
def getRevision(remote, remoteUrl, project, branch):
"""
Get latest commit sha1 for revision
with git ls-remote command for each project
without downloading the whole repo
"""
try:
with redirect_stdout(sys.stderr):
# return tuple (remote/project, branch, revision)
print('Fetching revision for {}/{} - {}...'.format(
remote, project, branch))
if is_sha1(branch):
return (remote + '/' + project, branch, branch)
g = git.cmd.Git()

headRef = branch
# v1.0^{} is the commit referring to tag v1.0
# git ls-remote returns the tag sha1 if left as is
if branch.startswith('refs/tags'):
headRef += '^{}'

url, revList = (
remote + '/' + project,
g.ls_remote(remoteUrl+'/'+project, headRef).split()
)

# convert revision list to revision dict:
# ['SHA1', 'refs/heads/XXXX', 'SHA1', 'refs/heads/YYYY']
# -> {'refs/heads/XXXX': 'SHA1', 'refs/heads/YYYY': 'SHA1'}
revDict = dict([(b, a)
for a, b in zip(revList[::2], revList[1::2])])

if branch.startswith('refs/tags'):
rev = headRef
else:
rev = 'refs/heads/' + branch

print('{} - {}: {}'.format(url, branch, revDict[rev]))

# return url, branch (without any suffix) and revision
return (url, branch, revDict[rev])
except Exception as e:
with redirect_stdout(sys.stderr):
print('Cannot fetch project {}/{}'.format(remoteUrl, project))
print(e)
with redirect_stdout(sys.stderr):
# return tuple (remote/project, branch, revision)
print('Fetching revision for {}/{} - {}...'.format(
remote, project, branch))
if is_sha1(branch):
return (remote + '/' + project, branch, branch)
g = git.cmd.Git()

headRef = branch
# v1.0^{} is the commit referring to tag v1.0
# git ls-remote returns the tag sha1 if left as is
if branch.startswith('refs/tags'):
headRef += '^{}'

url, revList = (
remote + '/' + project,
g.ls_remote(remoteUrl+'/'+project, headRef).split()
)

# convert revision list to revision dict:
# ['SHA1', 'refs/heads/XXXX', 'SHA1', 'refs/heads/YYYY']
# -> {'refs/heads/XXXX': 'SHA1', 'refs/heads/YYYY': 'SHA1'}
revDict = dict([(b, a)
for a, b in zip(revList[::2], revList[1::2])])

if branch.startswith('refs/tags'):
rev = headRef
else:
rev = 'refs/heads/' + branch

print('{} - {}: {}'.format(url, branch, revDict[rev]))

# return url, branch (without any suffix) and revision
return (url, branch, revDict[rev])


class SourceConfiguration(NamedTuple):
Expand Down
1 change: 1 addition & 0 deletions repo_resource/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ setuptools==74.1.2
gitrepo==2.32.2
ssh-agent-setup==2.0.1
GitPython==3.1.31
retrying==1.3.4
Loading