From cbf9c60f03ad4457b40447646be281c263f36b78 Mon Sep 17 00:00:00 2001 From: Evan Parker Date: Fri, 17 Nov 2017 10:32:45 -0800 Subject: [PATCH] Close out the rest of oauth compatibility issues (#186) * Update client to accommodate oauth2client>=4.0 This changes has been needed for a while now. The main blocker seems to be the use of locked_file for caching GCE credentials. I've added a simple multiprocess lockable file cache that uses a similar approach to that used in ouath2client's multiprocess file storage. Submission of this should close issue #162. * Update test runner setup --- .travis.yml | 44 ++++++++------ apitools/base/py/http_wrapper.py | 3 +- apitools/base/py/http_wrapper_test.py | 56 +++++++++++++++++- tox.ini | 84 ++++----------------------- 4 files changed, 91 insertions(+), 96 deletions(-) diff --git a/.travis.yml b/.travis.yml index 07b83aae..deaa5bf1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,32 @@ language: python sudo: false -env: - - TOX_ENV=py27 - - TOX_ENV=py27oldoauth2client - - TOX_ENV=py27newoauth2client - - TOX_ENV=py34 - - TOX_ENV=py35oauth2client15 - - TOX_ENV=py35oauth2client30 - - TOX_ENV=py35oauth2client41 - - TOX_ENV=lint +matrix: + include: + - python: "2.7" + env: TOX_ENV=lint + - python: "2.7" + env: TOX_ENV=py27-oauth2client1 + - python: "2.7" + env: TOX_ENV=py27-oauth2client2 + - python: "2.7" + env: TOX_ENV=py27-oauth2client3 + - python: "2.7" + env: TOX_ENV=py27-oauth2client4 + - python: "3.3" + env: TOX_ENV=py33-oauth2client4 + - python: "3.4" + env: TOX_ENV=py34-oauth2client4 + - python: "3.5" + env: TOX_ENV=py35-oauth2client1 + - python: "3.5" + env: TOX_ENV=py35-oauth2client2 + - python: "3.5" + env: TOX_ENV=py35-oauth2client3 + - python: "3.5" + env: TOX_ENV=py35-oauth2client4 install: - pip install tox - pip install . --allow-external argparse script: tox -e $TOX_ENV after_success: - - if [[ "${TOX_ENV}" == "py27" ]]; then tox -e coveralls; fi - -# Tweak for adding python3.5; see -# https://github.com/travis-ci/travis-ci/issues/4794 -addons: - apt: - sources: - - deadsnakes - packages: - - python3.5 + - if [[ "${TOX_ENV}" == "py27-oauth2client4" ]]; then tox -e coveralls; fi diff --git a/apitools/base/py/http_wrapper.py b/apitools/base/py/http_wrapper.py index c5fe225a..a3fe65cb 100644 --- a/apitools/base/py/http_wrapper.py +++ b/apitools/base/py/http_wrapper.py @@ -285,6 +285,7 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args): logging.debug('Response content was invalid (%s), retrying', retry_args.exc) elif (isinstance(retry_args.exc, TokenRefreshError) and + hasattr(retry_args.exc, 'status') and (retry_args.exc.status == TOO_MANY_REQUESTS or retry_args.exc.status >= 500)): logging.debug( @@ -300,7 +301,7 @@ def HandleExceptionsAndRebuildHttpConnections(retry_args): logging.debug('Response returned a retry-after header, retrying') retry_after = retry_args.exc.retry_after else: - raise # pylint: disable=misplaced-bare-raise + raise retry_args.exc RebuildHttpConnections(retry_args.http) logging.debug('Retrying request to url %s after exception %s', retry_args.http_request.url, retry_args.exc) diff --git a/apitools/base/py/http_wrapper_test.py b/apitools/base/py/http_wrapper_test.py index 5df107f1..ce4c03e6 100644 --- a/apitools/base/py/http_wrapper_test.py +++ b/apitools/base/py/http_wrapper_test.py @@ -17,7 +17,6 @@ import socket import httplib2 -import oauth2client from six.moves import http_client import unittest2 @@ -26,6 +25,15 @@ from apitools.base.py import exceptions from apitools.base.py import http_wrapper +# pylint: disable=ungrouped-imports +try: + from oauth2client.client import HttpAccessTokenRefreshError + from oauth2client.client import AccessTokenRefreshError + _TOKEN_REFRESH_STATUS_AVAILABLE = True +except ImportError: + from oauth2client.client import AccessTokenRefreshError + _TOKEN_REFRESH_STATUS_AVAILABLE = False + class _MockHttpRequest(object): @@ -57,6 +65,51 @@ def testRequestBodyUsesLengthProperty(self): def testRequestBodyWithLen(self): http_wrapper.Request(body='burrito') + @unittest2.skipIf(not _TOKEN_REFRESH_STATUS_AVAILABLE, + 'oauth2client<1.5 lacks HttpAccessTokenRefreshError.') + def testExceptionHandlerHttpAccessTokenError(self): + exception_arg = HttpAccessTokenRefreshError(status=503) + retry_args = http_wrapper.ExceptionRetryArgs( + http={'connections': {}}, http_request=_MockHttpRequest(), + exc=exception_arg, num_retries=0, max_retry_wait=0, + total_wait_sec=0) + + # Disable time.sleep for this handler as it is called with + # a minimum value of 1 second. + with patch('time.sleep', return_value=None): + http_wrapper.HandleExceptionsAndRebuildHttpConnections( + retry_args) + + @unittest2.skipIf(not _TOKEN_REFRESH_STATUS_AVAILABLE, + 'oauth2client<1.5 lacks HttpAccessTokenRefreshError.') + def testExceptionHandlerHttpAccessTokenErrorRaises(self): + exception_arg = HttpAccessTokenRefreshError(status=200) + retry_args = http_wrapper.ExceptionRetryArgs( + http={'connections': {}}, http_request=_MockHttpRequest(), + exc=exception_arg, num_retries=0, max_retry_wait=0, + total_wait_sec=0) + + # Disable time.sleep for this handler as it is called with + # a minimum value of 1 second. + with self.assertRaises(HttpAccessTokenRefreshError): + with patch('time.sleep', return_value=None): + http_wrapper.HandleExceptionsAndRebuildHttpConnections( + retry_args) + + def testExceptionHandlerAccessTokenErrorRaises(self): + exception_arg = AccessTokenRefreshError() + retry_args = http_wrapper.ExceptionRetryArgs( + http={'connections': {}}, http_request=_MockHttpRequest(), + exc=exception_arg, num_retries=0, max_retry_wait=0, + total_wait_sec=0) + + # Disable time.sleep for this handler as it is called with + # a minimum value of 1 second. + with self.assertRaises(AccessTokenRefreshError): + with patch('time.sleep', return_value=None): + http_wrapper.HandleExceptionsAndRebuildHttpConnections( + retry_args) + def testDefaultExceptionHandler(self): """Ensures exception handles swallows (retries)""" mock_http_content = 'content'.encode('utf8') @@ -68,7 +121,6 @@ def testDefaultExceptionHandler(self): socket.gaierror(), httplib2.ServerNotFoundError(), ValueError(), - oauth2client.client.HttpAccessTokenRefreshError(status=503), exceptions.RequestError(), exceptions.BadStatusCodeError( {'status': 503}, mock_http_content, 'url'), diff --git a/tox.ini b/tox.ini index 7a822355..b421af54 100644 --- a/tox.ini +++ b/tox.ini @@ -1,94 +1,30 @@ [tox] envlist = - py26 - py27 - py27oldoauth2client - py27newoauth2client - pypy - py34 - py35oauth2client15 - py35oauth2client30 - py35oauth2client41 - lint - cover + py26-oauth2client4 + py27-oauth2client{1,2,3,4} + py33-oauth2client41 + py34-oauth2client41 + py35-oauth2client{1,2,3,4} [testenv] deps = nose python-gflags==3.0.6 + oauth2client1: oauth2client<1.5dev + oauth2client2: oauth2client>=2,<=3dev + oauth2client3: oauth2client>=3,<=4dev + oauth2client4: oauth2client>=4,<=5dev commands = pip install google-apitools[testing] nosetests [] passenv = TRAVIS* -[testenv:py27newoauth2client] -commands = - pip install oauth2client==4.1.0 - {[testenv]commands} -deps = {[testenv]deps} - -[testenv:py27oldoauth2client] -commands = - pip install oauth2client==1.5.2 - {[testenv]commands} -deps = {[testenv]deps} - -[testenv:py34] -basepython = python3.4 -deps = - mock - nose - unittest2 -commands = nosetests [] - -[testenv:py35oauth2client15] -basepython = python3.5 -deps = - mock - nose - unittest2 -commands = - pip install oauth2client==1.5.2 - nosetests [] - -[testenv:py35oauth2client30] -basepython = python3.5 -deps = - mock - nose - unittest2 -commands = - pip install oauth2client==3.0.0 - nosetests [] - -[testenv:py35oauth2client41] -basepython = python3.5 -deps = - mock - nose - unittest2 -commands = - pip install oauth2client==4.1.0 - nosetests [] - -[testenv:py35] -basepython = python3.5 -deps = - mock - nose - unittest2 -commands = nosetests [] - -[pep8] -exclude = samples/*_sample/*/*,*/testdata/*,*.egg/,*.egg-info/,.*/,ez_setup.py,build -verbose = 1 - [testenv:lint] basepython = python2.7 commands = pip install six google-apitools[testing] - pep8 + pep8 apitools python run_pylint.py deps = pep8