diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index b83d01fd..8ebc11b2 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -4,7 +4,7 @@ Changelog
This file contains a brief summary of new features and dependency changes or
releases, in reverse chronological order.
-v0.0.6
+v0.1.0
------
New features
@@ -13,6 +13,13 @@ New features
* The GitHub username is no longer required upon obtaining organizations list.
* Provided ability to pass authentication token for github.com API requests via
environment variables.
+* Added ``-v`` argument support to enable verbose mode.
+
+Packaging changes
+~~~~~~~~~~~~~~~~~
+
+* Change the way to communicate with GitHub API. ```requests`` library no
+ longer used thanks to ``PyGithub``.
v0.0.5
------
diff --git a/README.rst b/README.rst
index 215fb3ce..4869202f 100644
--- a/README.rst
+++ b/README.rst
@@ -102,7 +102,7 @@ Usage
::
- gstore [-h] [--token TOKEN] [--org [ORG ...]] [target]
+ gstore [-h] [--token TOKEN] [--org [ORG ...]] [-v] [target]
**Positional arguments:**
@@ -118,13 +118,22 @@ Usage
``--token TOKEN``
An authentication token for github.com API requests. If not provided via
- argument environment variable ``GH_TOKEN`` or ``GITHUB_TOKEN`` will be used
- (in order of precedence). Setting these variables allows you not to not pass
- token directly via CLI argument and avoids storing it in the SHELL history.
+ CLI argument, then environment variable will be used. The order of searching
+ for a token in environment variables as follows (in order of precedence):
+
+ #. ``GH_TOKEN``, ``GITHUB_TOKEN``
+ #. ``GH_ENTERPRISE_TOKEN``, ``GITHUB_ENTERPRISE_TOKEN``
+
+ Setting these variables allows you not to not pass token directly via CLI
+ argument and avoids storing it in the Shell history.
``--org [ORG ...]``
Organizations you have access to (by default all).
+``-v``, ``--verbose``
+ Enable verbose mode. Causes Gstore to print debugging messages about its
+ progress in some cases.
+
Examples
~~~~~~~~
@@ -138,7 +147,8 @@ username, and the second one to get a list of user's organizations.
$ gstore --token "$TOKEN" ~/backup
Unless you set the ``GSTORE_DIR`` environment variable and don't provide
-*target*, Gstore will sync all the repositories to current working directory.:
+*target directory*, Gstore will sync all the repositories to current working
+directory.:
.. code-block:: bash
@@ -161,7 +171,7 @@ To get all repositories of a specific organization, just specify it as follows:
$ gstore --org Acme --token "$TOKEN" ~/backup
-To specify a *target* directory right after organization list use double dash
+To specify a *target directory* right after organization list use double dash
to signify the end of org option.:
.. code-block:: bash
@@ -181,17 +191,17 @@ Logging
-------
All informational and error messages produced by Gstore are sent directly to
-the standard streams of the operating system. Gstore doesn't have any special
-tools/options to setup logging to files. Such design was chosen deliberately
-to not increase Gstore complexity in those aspects where this is not clearly
-necessary, and also to simplify its administration by end users.
+the standard OS streams. Gstore doesn't have any special tools/options to setup
+logging to files. Such design was chosen deliberately to not increase Gstore
+complexity in those aspects where this is not clearly necessary, and also to
+simplify its administration by end users.
So, informational and error messages produced by Gstore are sent to two
separate streams:
-* The regular output is sent to standard output stream (STDOUT)
+* The regular output is sent to standard output stream (``STDOUT``)
* The error messages and the warning ones are sent to standard error stream
- (STDERR)
+ (``STDERR``)
The format of the messages generated by Gstore was chosen in such a way as to
preserve human readability, but at the same time to allow specialized tools to
diff --git a/gstore/__init__.py b/gstore/__init__.py
index be57a5a5..ec9881ec 100644
--- a/gstore/__init__.py
+++ b/gstore/__init__.py
@@ -14,7 +14,7 @@
# along with this file. If not, see .
__copyright__ = 'Copyright (C) 2020 Serghei Iakovlev'
-__version__ = '0.0.6'
+__version__ = '0.1.0'
__license__ = 'GPLv3+'
__author__ = 'Serghei Iakovlev'
__author_email__ = 'egrep@protonmail.ch'
diff --git a/gstore/args.py b/gstore/args.py
index acf4691e..305f96e0 100644
--- a/gstore/args.py
+++ b/gstore/args.py
@@ -19,17 +19,46 @@
from argparse import ArgumentParser
+def get_token_from_env():
+ """
+ Get authentication token for github.com from environment variables.
+
+ The order of searching for a token in environment variables:
+ * GH_TOKEN, GITHUB_TOKEN (in order of precedence)
+ * GH_ENTERPRISE_TOKEN, GITHUB_ENTERPRISE_TOKEN (in order of precedence)
+
+ :returns: An authentication token for github.com API requests
+ :rtype: str|None
+ """
+ token = None
+ toke_names = (
+ 'GH_TOKEN',
+ 'GITHUB_TOKEN',
+ 'GH_ENTERPRISE_TOKEN',
+ 'GITHUB_ENTERPRISE_TOKEN',
+ )
+
+ for name in toke_names:
+ token = env.get(name)
+ if token:
+ break
+
+ return token
+
+
def argparse():
p = ArgumentParser(
description="Synchronize organizations' repositories from GitHub.")
- p.add_argument('--token', dest='token',
- default=env.get('GH_TOKEN', env.get('GITHUB_TOKEN')),
- help='an authentication token for github.com API requests')
- p.add_argument('--org', dest='org', nargs='*',
- help='organizations you have access to (by default all)')
p.add_argument('target', nargs='?',
default=env.get('GSTORE_DIR', os.getcwd()),
help='base target to sync repos (e.g. folder on disk)')
+ p.add_argument('--token', dest='token', default=get_token_from_env(),
+ help='an authentication token for github.com API requests')
+ p.add_argument('--org', dest='org', nargs='*',
+ help='organizations you have access to (by default all)')
+ p.add_argument('-v', '--verbose', dest='verbose', action='store_true',
+ help='enable verbose mode')
+
return p.parse_args()
diff --git a/gstore/cli.py b/gstore/cli.py
index 37df61b9..b679fcba 100644
--- a/gstore/cli.py
+++ b/gstore/cli.py
@@ -13,23 +13,30 @@
# You should have received a copy of the GNU General Public License
# along with this file. If not, see .
+import logging
+
from .args import argparse
-from .client import get_orgs, get_repos, ensure_token_is_present
+from .client import Client
from .repo import sync
from .logger import setup_logger
def main():
ns = argparse()
+ setup_logger(verbose=ns.verbose)
+
+ logger = logging.getLogger('gstore.cli')
- setup_logger()
- ensure_token_is_present(ns.token)
+ try:
+ client = Client(ns.token)
- if ns.org is None:
- orgs = get_orgs(ns.token)
- else:
- orgs = ns.org
+ if ns.org is None:
+ orgs = client.get_orgs()
+ else:
+ orgs = ns.org
- for org in orgs:
- repos = get_repos(org, ns.token)
- sync(org, repos, ns.target)
+ for org in orgs:
+ repos = client.get_repos(org)
+ sync(org, repos, ns.target)
+ except Exception as e:
+ logger.critical(e)
diff --git a/gstore/client.py b/gstore/client.py
index 6d23303c..f6529fbd 100644
--- a/gstore/client.py
+++ b/gstore/client.py
@@ -13,124 +13,86 @@
# You should have received a copy of the GNU General Public License
# along with this file. If not, see .
-import requests
import logging
-API_URL = 'https://api.github.com'
-MAX_PAGES = 5000
-LOG = logging.getLogger('gstore.client')
+from github import Github
+from gstore import __version__
-class ClientApiException(Exception):
- pass
+USER_AGENT = 'Gstore/{}'.format(__version__)
+DEFAULT_BASE_URL = 'https://api.github.com'
+DEFAULT_TIMEOUT = 15
-def create_headers(token):
- return {
- 'Accept': 'application/vnd.github.v3+json',
- 'Authorization': 'token {}'.format(token)
- }
+class Client:
+ def __init__(
+ self,
+ token: str,
+ api_url=DEFAULT_BASE_URL,
+ timeout=DEFAULT_TIMEOUT,
+ ):
+ """
+ :param str token: Authentication token for github.com API requests
+ :param str api_url: Default base URL for github.com API requests
+ :param int timeout: Timeout for HTTP requests
+ :param str user_agent: Default user agent to make HTTP requests
+ """
-def collect_data(endpoint, params, headers, key):
- retval = []
- url = '{}{}'.format(API_URL, endpoint)
+ if not token:
+ raise ValueError(
+ 'GitHub token was not provided but it is mandatory')
- for i in range(MAX_PAGES):
- params['page'] = i + 1
- try:
- response = requests.get(url=url, params=params, headers=headers)
- response.raise_for_status()
+ self.github = Github(
+ login_or_token=token,
+ base_url=api_url,
+ timeout=timeout,
+ user_agent=USER_AGENT
+ )
- parsed_response = response.json()
+ self.logger = logging.getLogger('gstore.client')
- if len(parsed_response) == 0:
- break
+ def get_repos(self, org):
+ """
+ Getting organization repositories.
- for data in parsed_response:
- retval.append(data[key])
- except Exception as e:
- msg = 'Failed to perform API request, ' + str(e)
- raise ClientApiException(msg)
+ :param str org: User's organization
+ :return: A collection with repositories
+ :rtype: list
+ """
+ self.logger.info('Getting repositories for %s organization' % org)
- return retval
+ repos = self.github.get_organization(org).get_repos(
+ type='all',
+ sort='full_name'
+ )
+ self.logger.info('Total number of repositories for %s: %d' %
+ (org, repos.totalCount))
-def ensure_token_is_present(token):
- """
- Check GitHub Personal Access Token.
+ retval = []
+ for repo in repos:
+ retval.append(repo.name)
- :param str token: GitHub Personal Access Token
- """
- if not token:
- LOG.error('GitHub token was not provided but it is mandatory')
- exit(1)
+ return retval
+ def get_orgs(self):
+ """
+ Getting organizations for a user.
-def get_repos(org, token):
- """
- Getting organization repositories.
+ :returns: A collection with organizations
+ :rtype: list
+ """
+ self.logger.info('Getting organizations for a user')
- :param str org: User's organization
- :param str token: GitHub Personal Access Token
- :return: A collection with repositories
- :rtype: list
- """
- LOG.info('Getting organization repositories')
+ user = self.github.get_user()
+ orgs = user.get_orgs()
- endpoint = '/orgs/{}/repos'.format(org)
- params = {'per_page': 100, 'type': 'all', 'sort': 'full_name'}
- headers = create_headers(token)
+ self.logger.info('Total number of organizations for %s: %d' %
+ (user.login, orgs.totalCount))
- return collect_data(
- endpoint=endpoint,
- params=params,
- headers=headers,
- key='name')
+ retval = []
+ for org in orgs:
+ retval.append(org.login)
-
-def get_user(token):
- """
- Getting the authenticated user.
-
- :param str token: GitHub Personal Access Token
- :returns: A GitHUb username
- :rtype: str
- """
- LOG.info('Getting the authenticated user')
-
- url = '{}/user'.format(API_URL)
- headers = create_headers(token)
-
- try:
- response = requests.get(url=url, headers=headers)
- response.raise_for_status()
-
- parsed_response = response.json()
- return parsed_response['login']
- except Exception as e:
- msg = 'Failed to perform API request to get GitHub user, ' + str(e)
- raise ClientApiException(msg)
-
-
-def get_orgs(token):
- """
- Getting organizations for a user.
-
- :param str token: GitHub Personal Access Token
- :returns: A collection with organizations
- :rtype: list
- """
- LOG.info('Getting organizations for a user')
-
- user = get_user(token)
-
- endpoint = '/users/{}/orgs'.format(user)
- params = {'per_page': 100}
- headers = create_headers(token)
-
- return collect_data(
- endpoint=endpoint,
- params=params,
- headers=headers,
- key='login')
+ return retval
diff --git a/gstore/logger.py b/gstore/logger.py
index 81efd0e7..2ed02115 100644
--- a/gstore/logger.py
+++ b/gstore/logger.py
@@ -51,7 +51,7 @@ def filter(self, record: logging.LogRecord) -> bool:
return record.levelno == logging.INFO
-def setup_logger(*args, **kwargs):
+def setup_logger(verbose=False):
"""
Setup and return the root logger object for the application.
"""
@@ -62,22 +62,22 @@ def setup_logger(*args, **kwargs):
f = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
formatter = logging.Formatter(f)
- debug_handler = logging.StreamHandler(sys.stdout)
- debug_handler.setLevel(logging.DEBUG)
- debug_handler.addFilter(DebugFilter())
- debug_handler.setFormatter(formatter)
+ if verbose:
+ debug_handler = logging.StreamHandler(sys.stdout)
+ debug_handler.setLevel(logging.DEBUG)
+ debug_handler.addFilter(DebugFilter())
+ debug_handler.setFormatter(formatter)
+ root.addHandler(debug_handler)
info_handler = logging.StreamHandler(sys.stdout)
info_handler.setLevel(logging.INFO)
info_handler.addFilter(InfoFilter())
info_handler.setFormatter(formatter)
+ root.addHandler(info_handler)
error_handler = logging.StreamHandler(sys.stderr)
error_handler.setLevel(logging.WARNING)
error_handler.setFormatter(formatter)
-
- root.addHandler(debug_handler)
- root.addHandler(info_handler)
root.addHandler(error_handler)
return root
diff --git a/gstore/repo.py b/gstore/repo.py
index a91891e1..e9851ba3 100644
--- a/gstore/repo.py
+++ b/gstore/repo.py
@@ -101,7 +101,6 @@ def do_sync(org, repos, target):
if not os.path.exists(org_path):
os.makedirs(org_path)
- LOG.info('Total repos for %s: %d' % (org, len(repos)))
for repo_name in repos:
repo_path = os.path.join(org_path, repo_name)
diff --git a/requirements.txt b/requirements.txt
index 76717277..16fc5777 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,2 @@
-requests==2.23.0
+PyGithub==1.54.1
gitpython==3.0.6
diff --git a/setup.py b/setup.py
index 68e4bdbe..1c2abaed 100644
--- a/setup.py
+++ b/setup.py
@@ -185,7 +185,7 @@ def get_version_string(pkg_dir, pkg_name):
# Dependencies that are downloaded by pip on installation and why
install_requires=[
- 'requests>=2.23.0', # Web requests for GitHub API
+ 'PyGithub>=1.54.1', # Interact with GitHub objects
'gitpython>=3.0.6', # Interact with Git repositories
],