Update client to accommodate oauth2client>=4.0

.travis.yml

@@ -3,8 +3,11 @@ sudo: false
 env:
  - TOX_ENV=py27
  - TOX_ENV=py27oldoauth2client
+ - TOX_ENV=py27newoauth2client
  - TOX_ENV=py34
- - TOX_ENV=py35
+ - TOX_ENV=py35oauth2client15
+ - TOX_ENV=py35oauth2client30
+ - TOX_ENV=py35oauth2client41
  - TOX_ENV=lint
 install:
  - pip install tox

apitools/base/py/credentials_lib.py

@@ -17,17 +17,20 @@
 """Common credentials classes and constructors."""
 from __future__ import print_function
 
+import contextlib
 import datetime
 import json
 import os
 import threading
 import warnings
 
+import fasteners
 import httplib2
 import oauth2client
 import oauth2client.client
 from oauth2client import service_account
 from oauth2client import tools # for gflags declarations
+import six
 from six.moves import http_client
 from six.moves import urllib
 
@@ -45,14 +48,14 @@
  from oauth2client import gce
 
 try:
- from oauth2client.contrib import locked_file
+ from oauth2client.contrib import multiprocess_file_storage
+ _NEW_FILESTORE = True
 except ImportError:
- from oauth2client import locked_file
-
-try:
- from oauth2client.contrib import multistore_file
-except ImportError:
- from oauth2client import multistore_file
+ _NEW_FILESTORE = False
+ try:
+ from oauth2client.contrib import multistore_file
+ except ImportError:
+ from oauth2client import multistore_file
 
 try:
  import gflags
@@ -193,19 +196,6 @@ def ServiceAccountCredentialsFromP12File(
  user_agent=user_agent)
 
 
-def _EnsureFileExists(filename):
- """Touches a file; returns False on error, True on success."""
- if not os.path.exists(filename):
- old_umask = os.umask(0o177)
- try:
- open(filename, 'a+b').close()
- except OSError:
- return False
- finally:
- os.umask(old_umask)
- return True
-
-
 def _GceMetadataRequest(relative_url, use_metadata_ip=False):
  """Request the given url from the GCE metadata service."""
  if use_metadata_ip:
@@ -288,29 +278,20 @@ def _CheckCacheFileForMatch(self, cache_filename, scopes):
  'scopes': sorted(list(scopes)) if scopes else None,
  'svc_acct_name': self.__service_account_name,
  }
- with cache_file_lock:
- if _EnsureFileExists(cache_filename):
- cache_file = locked_file.LockedFile(
- cache_filename, 'r+b', 'rb')
- try:
- cache_file.open_and_lock()
- cached_creds_str = cache_file.file_handle().read()
- if cached_creds_str:
- # Cached credentials metadata dict.
- cached_creds = json.loads(cached_creds_str)
- if (creds['svc_acct_name'] ==
- cached_creds['svc_acct_name']):
- if (creds['scopes'] in
- (None, cached_creds['scopes'])):
- scopes = cached_creds['scopes']
- except KeyboardInterrupt:
- raise
- except: # pylint: disable=bare-except
- # Treat exceptions as a cache miss.
- pass
- finally:
- cache_file.unlock_and_close()
- return scopes
+ cache_file = _MultiProcessCacheFile(cache_filename)
+ try:
+ cached_creds_str = cache_file.LockedRead()
+ if not cached_creds_str:
+ return None
+ cached_creds = json.loads(cached_creds_str)
+ if creds['svc_acct_name'] == cached_creds['svc_acct_name']:
+ if creds['scopes'] in (None, cached_creds['scopes']):
+ return cached_creds['scopes']
+ except KeyboardInterrupt:
+ raise
+ except: # pylint: disable=bare-except
+ # Treat exceptions as a cache miss.
+ pass
 
  def _WriteCacheFile(self, cache_filename, scopes):
  """Writes the credential metadata to the cache file.
@@ -322,28 +303,18 @@ def _WriteCacheFile(self, cache_filename, scopes):
  cache_filename: Cache filename to check.
  scopes: Scopes for the desired credentials.
  """
- with cache_file_lock:
- if _EnsureFileExists(cache_filename):
- cache_file = locked_file.LockedFile(
- cache_filename, 'r+b', 'rb')
- try:
- cache_file.open_and_lock()
- if cache_file.is_locked():
- creds = { # Credentials metadata dict.
- 'scopes': sorted(list(scopes)),
- 'svc_acct_name': self.__service_account_name}
- cache_file.file_handle().write(
- json.dumps(creds, encoding='ascii'))
- # If it's not locked, the locking process will
- # write the same data to the file, so just
- # continue.
- except KeyboardInterrupt:
- raise
- except: # pylint: disable=bare-except
- # Treat exceptions as a cache miss.
- pass
- finally:
- cache_file.unlock_and_close()
+ # Credentials metadata dict.
+ creds = {'scopes': sorted(list(scopes)),
+ 'svc_acct_name': self.__service_account_name}
+ creds_str = json.dumps(creds)
+ cache_file = _MultiProcessCacheFile(cache_filename)
+ try:
+ cache_file.LockedWrite(creds_str)
+ except KeyboardInterrupt:
+ raise
+ except: # pylint: disable=bare-except
+ # Treat exceptions as a cache miss.
+ pass
 
  def _ScopesFromMetadataServer(self, scopes):
  """Returns instance scopes based on GCE metadata server."""
@@ -537,11 +508,18 @@ def _GetRunFlowFlags(args=None):
 # TODO(craigcitro): Switch this from taking a path to taking a stream.
 def CredentialsFromFile(path, client_info, oauth2client_args=None):
  """Read credentials from a file."""
- credential_store = multistore_file.get_credential_storage(
- path,
- client_info['client_id'],
- client_info['user_agent'],
- client_info['scope'])
+ user_agent = client_info['user_agent']
+ scope_key = client_info['scope']
+ if not isinstance(scope_key, six.string_types):
+ scope_key = ':'.join(scope_key)
+ storage_key = client_info['client_id'] + user_agent + scope_key
+
+ if _NEW_FILESTORE:
+ credential_store = multiprocess_file_storage.MultiprocessFileStorage(
+ path, storage_key)
+ else:
+ credential_store = multistore_file.get_credential_storage_custom_string_key( # noqa
+ path, storage_key)
  if hasattr(FLAGS, 'auth_local_webserver'):
  FLAGS.auth_local_webserver = False
  credentials = credential_store.get()
@@ -568,6 +546,101 @@ def CredentialsFromFile(path, client_info, oauth2client_args=None):
  return credentials
 
 
+class _MultiProcessCacheFile(object):
+ """Simple multithreading and multiprocessing safe cache file.
+
+ Notes on behavior:
+ * the fasteners.InterProcessLock object cannot reliably prevent threads
+ from double-acquiring a lock. A threading lock is used in addition to
+ the InterProcessLock. The threading lock is always acquired first and
+ released last.
+ * The interprocess lock will not deadlock. If a process can not acquire
+ the interprocess lock within `_lock_timeout` the call will return as
+ a cache miss or unsuccessful cache write.
+ """
+
+ _lock_timeout = 1
+ _encoding = 'utf-8'
+ _thread_lock = threading.Lock()
+
+ def __init__(self, filename):
+ self._file = None
+ self._filename = filename
+ self._process_lock = fasteners.InterProcessLock(
+ '{0}.lock'.format(filename))
+
+ @contextlib.contextmanager
+ def _ProcessLockAcquired(self):
+ """Context manager for process locks with timeout."""
+ try:
+ is_locked = self._process_lock.acquire(timeout=self._lock_timeout)
+ yield is_locked
+ finally:
+ if is_locked:
+ self._process_lock.release()
+
+ def LockedRead(self):
+ """Acquire an interprocess lock and dump cache contents.
+
+ This method safely acquires the locks then reads a string
+ from the cache file. If the file does not exist and cannot
+ be created, it will return None. If the locks cannot be
+ acquired, this will also return None.
+
+ Returns:
+ cache data - string if present, None on failure.
+ """
+ file_contents = None
+ with self._thread_lock:
+ if not self._EnsureFileExists():
+ return None
+ with self._ProcessLockAcquired() as acquired_plock:
+ if not acquired_plock:
+ return None
+ with open(self._filename, 'rb') as f:
+ file_contents = f.read().decode(encoding=self._encoding)
+ return file_contents
+
+ def LockedWrite(self, cache_data):
+ """Acquire an interprocess lock and write a string.
+
+ This method safely acquires the locks then writes a string
+ to the cache file. If the string is written successfully
+ the function will return True, if the write fails for any
+ reason it will return False.
+
+ Args:
+ cache_data: string or bytes to write.
+
+ Returns:
+ bool: success
+ """
+ if isinstance(cache_data, six.text_type):
+ cache_data = cache_data.encode(encoding=self._encoding)
+
+ with self._thread_lock:
+ if not self._EnsureFileExists():
+ return False
+ with self._ProcessLockAcquired() as acquired_plock:
+ if not acquired_plock:
+ return False
+ with open(self._filename, 'wb') as f:
+ f.write(cache_data)
+ return True
+
+ def _EnsureFileExists(self):
+ """Touches a file; returns False on error, True on success."""
+ if not os.path.exists(self._filename):
+ old_umask = os.umask(0o177)
+ try:
+ open(self._filename, 'a+b').close()
+ except OSError:
+ return False
+ finally:
+ os.umask(old_umask)
+ return True
+
+
 # TODO(craigcitro): Push this into oauth2client.
 def GetUserinfo(credentials, http=None): # pylint: disable=invalid-name
  """Get the userinfo associated with the given credentials.