Merge pull request #3 from sergeyklay/feature/PyGithub

Change GitHub API library to PyGithub

CHANGELOG.rst

@@ -4,7 +4,7 @@ Changelog
 This file contains a brief summary of new features and dependency changes or
 releases, in reverse chronological order.
 
-v0.0.6
+v0.1.0
 ------
 
 New features
@@ -13,6 +13,13 @@ New features
 * The GitHub username is no longer required upon obtaining organizations list.
 * Provided ability to pass authentication token for github.com API requests via
   environment variables.
+* Added ``-v`` argument support to enable verbose mode.
+
+Packaging changes
+~~~~~~~~~~~~~~~~~
+
+* Change the way to communicate with GitHub API. ```requests`` library no
+  longer used thanks to ``PyGithub``.
 
 v0.0.5
 ------

README.rst

@@ -102,7 +102,7 @@ Usage
 
 ::
 
-   gstore [-h] [--token TOKEN] [--org [ORG ...]] [target]
+   gstore [-h] [--token TOKEN] [--org [ORG ...]] [-v] [target]
 
 **Positional arguments:**
 
@@ -118,13 +118,22 @@ Usage
 
 ``--token TOKEN``
   An authentication token for github.com API requests. If not provided via
-  argument environment variable ``GH_TOKEN`` or ``GITHUB_TOKEN`` will be used
-  (in order of precedence). Setting these variables allows you not to not pass
-  token directly via CLI argument and avoids storing it in the SHELL history.
+  CLI argument, then environment variable will be used. The order of searching
+  for a token in environment variables as follows (in order of precedence):
+
+  #. ``GH_TOKEN``, ``GITHUB_TOKEN``
+  #. ``GH_ENTERPRISE_TOKEN``, ``GITHUB_ENTERPRISE_TOKEN``
+
+  Setting these variables allows you not to not pass token directly via CLI
+  argument and avoids storing it in the Shell history.
 
 ``--org [ORG ...]``
   Organizations you have access to (by default all).
 
+``-v``, ``--verbose``
+  Enable verbose mode. Causes Gstore to print debugging messages about its
+  progress in some cases.
+
 Examples
 ~~~~~~~~
 
@@ -138,7 +147,8 @@ username, and the second one to get a list of user's organizations.
    $ gstore --token "$TOKEN" ~/backup
 
 Unless you set the ``GSTORE_DIR`` environment variable and don't provide
-*target*, Gstore will sync all the repositories to current working directory.:
+*target directory*, Gstore will sync all the repositories to current working
+directory.:
 
 .. code-block:: bash
 
@@ -161,7 +171,7 @@ To get all repositories of a specific organization, just specify it as follows:
 
    $ gstore --org Acme --token "$TOKEN" ~/backup
 
-To specify a *target* directory right after organization list use double dash
+To specify a *target directory* right after organization list use double dash
 to signify the end of org option.:
 
 .. code-block:: bash
@@ -181,17 +191,17 @@ Logging
 -------
 
 All informational and error messages produced by Gstore are sent directly to
-the standard streams of the operating system. Gstore doesn't have any special
-tools/options to setup logging to files. Such design was chosen deliberately
-to not increase Gstore complexity in those aspects where this is not clearly
-necessary, and also to simplify its administration by end users.
+the standard OS streams. Gstore doesn't have any special tools/options to setup
+logging to files. Such design was chosen deliberately to not increase Gstore
+complexity in those aspects where this is not clearly necessary, and also to
+simplify its administration by end users.
 
 So, informational and error messages produced by Gstore are sent to two
 separate streams:
 
-* The regular output is sent to standard output stream (STDOUT)
+* The regular output is sent to standard output stream (``STDOUT``)
 * The error messages and the warning ones are sent to standard error stream
-  (STDERR)
+  (``STDERR``)
 
 The format of the messages generated by Gstore was chosen in such a way as to
 preserve human readability, but at the same time to allow specialized tools to

gstore/__init__.py

@@ -14,7 +14,7 @@
 # along with this file.  If not, see <https://www.gnu.org/licenses/>.
 
 __copyright__ = 'Copyright (C) 2020 Serghei Iakovlev'
-__version__ = '0.0.6'
+__version__ = '0.1.0'
 __license__ = 'GPLv3+'
 __author__ = 'Serghei Iakovlev'
 __author_email__ = '[email protected]'

gstore/args.py

@@ -19,17 +19,46 @@
 from argparse import ArgumentParser
 
 
+def get_token_from_env():
+    """
+    Get authentication token for github.com from environment variables.
+
+    The order of searching for a token in environment variables:
+    * GH_TOKEN, GITHUB_TOKEN (in order of precedence)
+    * GH_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_TOKEN (in order of precedence)
+
+    :returns: An authentication token for github.com API requests
+    :rtype: str|None
+    """
+    token = None
+    toke_names = (
+        'GH_TOKEN',
+        'GITHUB_TOKEN',
+        'GH_ENTERPRISE_TOKEN',
+        'GITHUB_ENTERPRISE_TOKEN',
+    )
+
+    for name in toke_names:
+        token = env.get(name)
+        if token:
+            break
+
+    return token
+
+
 def argparse():
     p = ArgumentParser(
         description="Synchronize organizations' repositories from GitHub.")
 
-    p.add_argument('--token', dest='token',
-                   default=env.get('GH_TOKEN', env.get('GITHUB_TOKEN')),
-                   help='an authentication token for github.com API requests')
-    p.add_argument('--org', dest='org', nargs='*',
-                   help='organizations you have access to (by default all)')
     p.add_argument('target', nargs='?',
                    default=env.get('GSTORE_DIR', os.getcwd()),
                    help='base target to sync repos (e.g. folder on disk)')
 
+    p.add_argument('--token', dest='token', default=get_token_from_env(),
+                   help='an authentication token for github.com API requests')
+    p.add_argument('--org', dest='org', nargs='*',
+                   help='organizations you have access to (by default all)')
+    p.add_argument('-v', '--verbose', dest='verbose', action='store_true',
+                   help='enable verbose mode')
+
     return p.parse_args()

gstore/cli.py

@@ -13,23 +13,30 @@
 # You should have received a copy of the GNU General Public License
 # along with this file.  If not, see <https://www.gnu.org/licenses/>.
 
+import logging
+
 from .args import argparse
-from .client import get_orgs, get_repos, ensure_token_is_present
+from .client import Client
 from .repo import sync
 from .logger import setup_logger
 
 
 def main():
     ns = argparse()
+    setup_logger(verbose=ns.verbose)
+
+    logger = logging.getLogger('gstore.cli')
 
-    setup_logger()
-    ensure_token_is_present(ns.token)
+    try:
+        client = Client(ns.token)
 
-    if ns.org is None:
-        orgs = get_orgs(ns.token)
-    else:
-        orgs = ns.org
+        if ns.org is None:
+            orgs = client.get_orgs()
+        else:
+            orgs = ns.org
 
-    for org in orgs:
-        repos = get_repos(org, ns.token)
-        sync(org, repos, ns.target)
+        for org in orgs:
+            repos = client.get_repos(org)
+            sync(org, repos, ns.target)
+    except Exception as e:
+        logger.critical(e)

gstore/client.py

@@ -13,124 +13,86 @@
 # You should have received a copy of the GNU General Public License
 # along with this file.  If not, see <https://www.gnu.org/licenses/>.
 
-import requests
 import logging
 
-API_URL = 'https://api.github.com'
-MAX_PAGES = 5000
-LOG = logging.getLogger('gstore.client')
+from github import Github
+from gstore import __version__
 
 
-class ClientApiException(Exception):
-    pass
+USER_AGENT = 'Gstore/{}'.format(__version__)
 
+DEFAULT_BASE_URL = 'https://api.github.com'
+DEFAULT_TIMEOUT = 15
 
-def create_headers(token):
-    return {
-        'Accept': 'application/vnd.github.v3+json',
-        'Authorization': 'token {}'.format(token)
-    }
 
+class Client:
+    def __init__(
+            self,
+            token: str,
+            api_url=DEFAULT_BASE_URL,
+            timeout=DEFAULT_TIMEOUT,
+    ):
+        """
+        :param str token: Authentication token for github.com API requests
+        :param str api_url: Default base URL for github.com API requests
+        :param int timeout: Timeout for HTTP requests
+        :param str user_agent: Default user agent to make HTTP requests
+        """
 
-def collect_data(endpoint, params, headers, key):
-    retval = []
-    url = '{}{}'.format(API_URL, endpoint)
+        if not token:
+            raise ValueError(
+                'GitHub token was not provided but it is mandatory')
 
-    for i in range(MAX_PAGES):
-        params['page'] = i + 1
-        try:
-            response = requests.get(url=url, params=params, headers=headers)
-            response.raise_for_status()
+        self.github = Github(
+            login_or_token=token,
+            base_url=api_url,
+            timeout=timeout,
+            user_agent=USER_AGENT
+        )
 
-            parsed_response = response.json()
+        self.logger = logging.getLogger('gstore.client')
 
-            if len(parsed_response) == 0:
-                break
+    def get_repos(self, org):
+        """
+        Getting organization repositories.
 
-            for data in parsed_response:
-                retval.append(data[key])
-        except Exception as e:
-            msg = 'Failed to perform API request, ' + str(e)
-            raise ClientApiException(msg)
+        :param str org: User's organization
+        :return: A collection with repositories
+        :rtype: list
+        """
+        self.logger.info('Getting repositories for %s organization' % org)
 
-    return retval
+        repos = self.github.get_organization(org).get_repos(
+            type='all',
+            sort='full_name'
+        )
 
+        self.logger.info('Total number of repositories for %s: %d' %
+                         (org, repos.totalCount))
 
-def ensure_token_is_present(token):
-    """
-    Check GitHub Personal Access Token.
+        retval = []
+        for repo in repos:
+            retval.append(repo.name)
 
-    :param str token: GitHub Personal Access Token
-    """
-    if not token:
-        LOG.error('GitHub token was not provided but it is mandatory')
-        exit(1)
+        return retval
 
+    def get_orgs(self):
+        """
+        Getting organizations for a user.
 
-def get_repos(org, token):
-    """
-    Getting organization repositories.
+        :returns: A collection with organizations
+        :rtype: list
+        """
+        self.logger.info('Getting organizations for a user')
 
-    :param str org: User's organization
-    :param str token: GitHub Personal Access Token
-    :return: A collection with repositories
-    :rtype: list
-    """
-    LOG.info('Getting organization repositories')
+        user = self.github.get_user()
+        orgs = user.get_orgs()
 
-    endpoint = '/orgs/{}/repos'.format(org)
-    params = {'per_page': 100, 'type': 'all', 'sort': 'full_name'}
-    headers = create_headers(token)
+        self.logger.info('Total number of organizations for %s: %d' %
+                         (user.login, orgs.totalCount))
 
-    return collect_data(
-        endpoint=endpoint,
-        params=params,
-        headers=headers,
-        key='name')
+        retval = []
+        for org in orgs:
+            retval.append(org.login)
 
-
-def get_user(token):
-    """
-    Getting the authenticated user.
-
-    :param str token: GitHub Personal Access Token
-    :returns: A GitHUb username
-    :rtype: str
-    """
-    LOG.info('Getting the authenticated user')
-
-    url = '{}/user'.format(API_URL)
-    headers = create_headers(token)
-
-    try:
-        response = requests.get(url=url, headers=headers)
-        response.raise_for_status()
-
-        parsed_response = response.json()
-        return parsed_response['login']
-    except Exception as e:
-        msg = 'Failed to perform API request to get GitHub user, ' + str(e)
-        raise ClientApiException(msg)
-
-
-def get_orgs(token):
-    """
-    Getting organizations for a user.
-
-    :param str token: GitHub Personal Access Token
-    :returns: A collection with organizations
-    :rtype: list
-    """
-    LOG.info('Getting organizations for a user')
-
-    user = get_user(token)
-
-    endpoint = '/users/{}/orgs'.format(user)
-    params = {'per_page': 100}
-    headers = create_headers(token)
-
-    return collect_data(
-        endpoint=endpoint,
-        params=params,
-        headers=headers,
-        key='login')
+        return retval