From bfa6fd89d7e3687a057658c7aaa073664de97699 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?David=20Roz=C3=A9?= <droze@baylibre.com>
Date: Fri, 11 Oct 2024 15:31:46 +0200
Subject: [PATCH] common: git retry feature
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We may face some "too many requests" failures (429 error) with some
servers. It's better to retry the failed git command on a single repo
rather than restart the whole process later.

Signed-off-by: David Rozé <droze@baylibre.com>
---
 repo_resource/common.py        | 91 +++++++++++++++++++---------------
 repo_resource/requirements.txt |  1 +
 2 files changed, 53 insertions(+), 39 deletions(-)

diff --git a/repo_resource/common.py b/repo_resource/common.py
index e23c6ef..7ca8ad2 100644
--- a/repo_resource/common.py
+++ b/repo_resource/common.py
@@ -21,6 +21,7 @@
 from typing import NamedTuple
 from urllib.parse import urlparse
 from multiprocessing import Pool
+from retrying import retry
 
 import ssh_agent_setup
 from repo import manifest_xml
@@ -28,6 +29,8 @@
 
 
 DEFAULT_CHECK_JOBS = 2
+GIT_LS_REMOTE_MAX_RETRIES = 3
+GIT_LS_REMOTE_WAIT = 2000  # 2s
 CACHEDIR = Path('/tmp/repo-resource-cache')
 SHA1_PATTERN = re.compile(r'^[0-9a-f]{40}$')
 EXCLUDE_ATTRS = {'dest-branch', 'upstream'}
@@ -88,51 +91,61 @@ def multi_run_wrapper(args):
     return getRevision(*args)
 
 
+def retry_getRevision(exception):
+    """Return True if we should retry, raise exception otherwise"""
+    with redirect_stdout(sys.stderr):
+        print(exception)
+        if "HTTP 429" not in str(exception).upper():
+            raise str(exception)
+        print('git ls-remote: sleeping {:.1f} seconds before retrying'.format(
+            GIT_LS_REMOTE_WAIT // 1000)
+        )
+        return True
+
+
+@retry(retry_on_exception=retry_getRevision,
+       stop_max_attempt_number=GIT_LS_REMOTE_MAX_RETRIES,
+       wait_fixed=GIT_LS_REMOTE_WAIT)
 def getRevision(remote, remoteUrl, project, branch):
     """
     Get latest commit sha1 for revision
     with git ls-remote command for each project
     without downloading the whole repo
     """
-    try:
-        with redirect_stdout(sys.stderr):
-            # return tuple (remote/project, branch, revision)
-            print('Fetching revision for {}/{} - {}...'.format(
-                remote, project, branch))
-            if is_sha1(branch):
-                return (remote + '/' + project, branch, branch)
-            g = git.cmd.Git()
-
-            headRef = branch
-            # v1.0^{} is the commit referring to tag v1.0
-            # git ls-remote returns the tag sha1 if left as is
-            if branch.startswith('refs/tags'):
-                headRef += '^{}'
-
-            url, revList = (
-                remote + '/' + project,
-                g.ls_remote(remoteUrl+'/'+project, headRef).split()
-            )
-
-            # convert revision list to revision dict:
-            # ['SHA1', 'refs/heads/XXXX', 'SHA1', 'refs/heads/YYYY']
-            # -> {'refs/heads/XXXX': 'SHA1', 'refs/heads/YYYY': 'SHA1'}
-            revDict = dict([(b, a)
-                            for a, b in zip(revList[::2], revList[1::2])])
-
-            if branch.startswith('refs/tags'):
-                rev = headRef
-            else:
-                rev = 'refs/heads/' + branch
-
-            print('{} - {}: {}'.format(url, branch, revDict[rev]))
-
-            # return url, branch (without any suffix) and revision
-            return (url, branch, revDict[rev])
-    except Exception as e:
-        with redirect_stdout(sys.stderr):
-            print('Cannot fetch project {}/{}'.format(remoteUrl, project))
-            print(e)
+    with redirect_stdout(sys.stderr):
+        # return tuple (remote/project, branch, revision)
+        print('Fetching revision for {}/{} - {}...'.format(
+            remote, project, branch))
+        if is_sha1(branch):
+            return (remote + '/' + project, branch, branch)
+        g = git.cmd.Git()
+
+        headRef = branch
+        # v1.0^{} is the commit referring to tag v1.0
+        # git ls-remote returns the tag sha1 if left as is
+        if branch.startswith('refs/tags'):
+            headRef += '^{}'
+
+        url, revList = (
+            remote + '/' + project,
+            g.ls_remote(remoteUrl+'/'+project, headRef).split()
+        )
+
+        # convert revision list to revision dict:
+        # ['SHA1', 'refs/heads/XXXX', 'SHA1', 'refs/heads/YYYY']
+        # -> {'refs/heads/XXXX': 'SHA1', 'refs/heads/YYYY': 'SHA1'}
+        revDict = dict([(b, a)
+                        for a, b in zip(revList[::2], revList[1::2])])
+
+        if branch.startswith('refs/tags'):
+            rev = headRef
+        else:
+            rev = 'refs/heads/' + branch
+
+        print('{} - {}: {}'.format(url, branch, revDict[rev]))
+
+        # return url, branch (without any suffix) and revision
+        return (url, branch, revDict[rev])
 
 
 class SourceConfiguration(NamedTuple):
diff --git a/repo_resource/requirements.txt b/repo_resource/requirements.txt
index 45ffbba..c1c2e05 100644
--- a/repo_resource/requirements.txt
+++ b/repo_resource/requirements.txt
@@ -2,3 +2,4 @@ setuptools==74.1.2
 gitrepo==2.32.2
 ssh-agent-setup==2.0.1
 GitPython==3.1.31
+retrying==1.3.4