From e48c9e37d7f542100a8ca331a9ff097280135a63 Mon Sep 17 00:00:00 2001 From: Jeffrey Meadows Date: Tue, 13 Oct 2015 17:47:40 -0700 Subject: [PATCH 1/5] Add network and auth subclasses. -Adds LoggingNetwork to allow clients to see what network calls the SDK is making. -Adds RemoteOAuth2 to allow auth to occur in a different process or machine from the SDK. -Adds RedisManagedOAuth2 to allow token storage in Redis, to enable multiple processes or machines to share tokens. --- HISTORY.rst | 2 + README.rst | 24 +++++++++ boxsdk/auth/oauth2.py | 25 ++++++++-- boxsdk/auth/redis_managed_oauth2.py | 76 +++++++++++++++++++++++++++++ boxsdk/auth/remote_oauth2.py | 26 ++++++++++ boxsdk/network/logging_network.py | 70 ++++++++++++++++++++++++++ boxsdk/util/log.py | 37 ++++++++++++++ requirements.txt | 1 + setup.py | 3 +- 9 files changed, 258 insertions(+), 6 deletions(-) create mode 100644 boxsdk/auth/redis_managed_oauth2.py create mode 100644 boxsdk/auth/remote_oauth2.py create mode 100644 boxsdk/network/logging_network.py create mode 100644 boxsdk/util/log.py diff --git a/HISTORY.rst b/HISTORY.rst index e5cf7ee35..05e61201e 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -9,6 +9,8 @@ Upcoming - CPython 3.5 support. - Support for cryptography>=1.0 on PyPy 2.6. - Travis CI testing for CPython 3.5 and PyPy 2.6.0. +- Added a logging network class that logs requests and responses. +- Added new options for auth classes, including storing tokens in Redis and storing them on a remote server. 1.2.1 (2015-07-22) ++++++++++++++++++ diff --git a/README.rst b/README.rst index 77ce25e75..e7fecb95a 100644 --- a/README.rst +++ b/README.rst @@ -282,6 +282,30 @@ These users can then be authenticated: Requests made with `ned_client` (or objects returned from `ned_client`'s methods) will be performed on behalf of the newly created app user. +Other Auth Options +------------------ + +For advanced uses of the SDK, two additional auth classes are provided: + +- `RemoteOAuth2`: Allows use of the SDK on clients without access to your application's client secret. Instead, you +provide a `retrieve_access_token` callback. That callback should perform the token refresh, perhaps on your server +that does have access to the client secret. +- `RedisManagedOAuth2`: Stores access and refresh tokens in Redis. This allows multiple processes (possibly spanning +multiple machines) to share access tokens while synchronizing token refresh. This could be useful for a multiprocess +web server, for example. + +Other Network Options +--------------------- + +For more insight into the network calls the SDK is making, you can use the `LoggingNetwork` class. This class logs +information about network requests and responses made to the Box API. + +.. code-block:: python + + from boxsdk import Client + from boxsdk.network.logging_network import LoggingNetwork + + client = Client(oauth, network_layer=LoggingNetwork()) Contributing ------------ diff --git a/boxsdk/auth/oauth2.py b/boxsdk/auth/oauth2.py index 0d5203582..1cd3e95c9 100644 --- a/boxsdk/auth/oauth2.py +++ b/boxsdk/auth/oauth2.py @@ -65,7 +65,7 @@ def __init__( """ self._client_id = client_id self._client_secret = client_secret - self._store_tokens = store_tokens + self._store_tokens_callback = store_tokens self._access_token = access_token self._refresh_token = refresh_token self._network_layer = network_layer if network_layer else DefaultNetwork() @@ -158,6 +158,17 @@ def _refresh(self, access_token): return self.send_token_request(data, access_token) + def _get_tokens(self): + """ + Get the current access and refresh tokens. + + :return: + Tuple containing the current access token and refresh token. + :rtype: + `tuple` of (`unicode`, `unicode`) + """ + return self._access_token, self._refresh_token + def refresh(self, access_token_to_refresh): """ Refresh the access token and the refresh token and return the access_token, refresh_token tuple. The access @@ -169,16 +180,17 @@ def refresh(self, access_token_to_refresh): `unicode` """ with self._refresh_lock: + access_token, refresh_token = self._get_tokens() # The lock here is for handling that case that multiple requests fail, due to access token expired, at the # same time to avoid multiple session renewals. - if access_token_to_refresh == self._access_token: + if access_token_to_refresh == access_token: # If the active access token is the same as the token needs to be refreshed, we make the request to # refresh the token. return self._refresh(access_token_to_refresh) else: # If the active access token (self._access_token) is not the same as the token needs to be refreshed, # it means the expired token has already been refreshed. Simply return the current active tokens. - return self._access_token, self._refresh_token + return access_token, refresh_token @staticmethod def _get_state_csrf_token(): @@ -195,6 +207,10 @@ def _get_state_csrf_token(): ascii_len = len(ascii_alphabet) return 'box_csrf_token_' + ''.join(ascii_alphabet[int(system_random.random() * ascii_len)] for _ in range(16)) + def _store_tokens(self, access_token, refresh_token): + if self._store_tokens_callback is not None: + self._store_tokens_callback(access_token, refresh_token) + def send_token_request(self, data, access_token, expect_refresh_token=True): """ Send the request to acquire or refresh an access token. @@ -231,6 +247,5 @@ def send_token_request(self, data, access_token, expect_refresh_token=True): raise BoxOAuthException(network_response.status_code, network_response.content, url, 'POST') except (ValueError, KeyError): raise BoxOAuthException(network_response.status_code, network_response.content, url, 'POST') - if self._store_tokens: - self._store_tokens(self._access_token, self._refresh_token) + self._store_tokens(self._access_token, self._refresh_token) return self._access_token, self._refresh_token diff --git a/boxsdk/auth/redis_managed_oauth2.py b/boxsdk/auth/redis_managed_oauth2.py new file mode 100644 index 000000000..921654941 --- /dev/null +++ b/boxsdk/auth/redis_managed_oauth2.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +from __future__ import unicode_literals +from redis import StrictRedis +from redis.lock import Lock +from uuid import uuid4 +from boxsdk import JWTAuth, OAuth2 + + +class RedisManagedOAuth2Mixin(OAuth2): + """ + Box SDK OAuth2 subclass. + Allows for storing auth tokens in redis. + + :param unique_id: + An identifier for this auth object. Auth instances which wish to share tokens must use the same ID. + :type unique_id: + `unicode` + :param redis_server: + An instance of a Redis server, configured to talk to Redis. + :type redis_server: + :class:`Redis` + """ + def __init__(self, unique_id=uuid4(), redis_server=None, *args, **kwargs): + self._unique_id = unique_id + self._redis_server = redis_server or StrictRedis() + super(RedisManagedOAuth2Mixin, self).__init__(*args, **kwargs) + if self._access_token is None: + self._update_current_tokens() + self._refresh_lock = Lock(redis=self._redis_server, name='{0}_lock'.format(self._unique_id)) + + def _update_current_tokens(self): + self._access_token, self._refresh_token = self._redis_server.hvals(self._unique_id) or (None, None) + + @property + def unique_id(self): + """ + Get the unique ID used by this auth instance. Other instances can share tokens with this instance + if they share the ID with this instance. + """ + return self._unique_id + + def _get_tokens(self): + """ + Base class override. + Gets the latest tokens from redis before returning them. + """ + self._update_current_tokens() + return super(RedisManagedOAuth2Mixin, self)._get_tokens() + + def send_token_request(self, *args, **kwargs): + """ + Base class override. + Saves the refreshed tokens in redis. + """ + tokens = super(RedisManagedOAuth2Mixin, self).send_token_request(*args, **kwargs) + self._redis_server.hmset(self._unique_id, {'access': tokens[0], 'refresh': tokens[1]}) + return tokens + + +class RedisManagedOAuth2(RedisManagedOAuth2Mixin): + """ + OAuth2 subclass which uses Redis to manage tokens. + """ + pass + + +class RedisManagedJWTAuth(RedisManagedOAuth2Mixin, JWTAuth): + """ + JWT Auth subclass which uses Redis to manage access tokens. + """ + def _auth_with_jwt(self, sub, sub_type): + """ + Base class override. Returns the access token in a tuple to match the OAuth2 interface. + """ + return super(RedisManagedJWTAuth, self)._auth_with_jwt(sub, sub_type), None diff --git a/boxsdk/auth/remote_oauth2.py b/boxsdk/auth/remote_oauth2.py new file mode 100644 index 000000000..5cf07e3e2 --- /dev/null +++ b/boxsdk/auth/remote_oauth2.py @@ -0,0 +1,26 @@ +# coding: utf-8 + +from __future__ import unicode_literals +from boxsdk import OAuth2 + + +class RemoteOAuth2Mixin(OAuth2): + """ + Box SDK OAuth2 subclass. + Allows for storing auth tokens remotely. + + :param retrieve_access_token: + Callback to exchange an existing access token for a new one. + :type retrieve_access_token: + `callable` of `unicode` => `unicode` + """ + def __init__(self, retrieve_access_token=None, *args, **kwargs): + self._retrieve_access_token = retrieve_access_token + super(RemoteOAuth2Mixin, self).__init__(*args, **kwargs) + + def _refresh(self, access_token): + """ + Base class override. Ask the remote host for a new token. + """ + self._access_token = self._retrieve_access_token(access_token) + return self._access_token, None diff --git a/boxsdk/network/logging_network.py b/boxsdk/network/logging_network.py new file mode 100644 index 000000000..53799432e --- /dev/null +++ b/boxsdk/network/logging_network.py @@ -0,0 +1,70 @@ +# coding: utf-8 + +from __future__ import unicode_literals +from pprint import pformat +from boxsdk.network.default_network import DefaultNetwork +from boxsdk.util.log import setup_logging + + +class LoggingNetwork(DefaultNetwork): + """ + SDK Network subclass that logs requests and responses. + """ + def __init__(self, logger=None): + """ + :param logger: + The logger to use. If you instantiate this class more than once, you should use the same logger + to avoid duplicate log entries. + :type logger: + :class:`Logger` + """ + super(LoggingNetwork, self).__init__() + self._logger = logger or setup_logging(name='network') + + @property + def logger(self): + return self._logger + + def _log_request(self, method, url, **kwargs): + """ + Logs information about the Box API request. + + :param method: + The HTTP verb that should be used to make the request. + :type method: + `unicode` + :param url: + The URL for the request. + :type url: + `unicode` + :param access_token: + The OAuth2 access token used to authorize the request. + :type access_token: + `unicode` + """ + self._logger.info('\x1b[36m%s %s %s\x1b[0m', method, url, pformat(kwargs)) + + def _log_response(self, response): + """ + Logs information about the Box API response. + + :param response: The Box API response. + """ + if response.ok: + self._logger.info('\x1b[32m%s\x1b[0m', response.content) + else: + self._logger.warning( + '\x1b[31m%s\n%s\n%s\n\x1b[0m', + response.status_code, + response.headers, + pformat(response.content), + ) + + def request(self, method, url, access_token, **kwargs): + """ + Base class override. Logs information about an API request and response in addition to making the request. + """ + self._log_request(method, url, **kwargs) + response = super(LoggingNetwork, self).request(method, url, access_token, **kwargs) + self._log_response(response) + return response diff --git a/boxsdk/util/log.py b/boxsdk/util/log.py new file mode 100644 index 000000000..f5f11b2fa --- /dev/null +++ b/boxsdk/util/log.py @@ -0,0 +1,37 @@ +# coding: utf-8 + +from __future__ import unicode_literals +import logging +import sys + + +def setup_logging(stream_or_file=None, debug=False, name=None): + """ + Create a logger for communicating with the user or writing to log files. + By default, creates a root logger that prints to stdout. + + :param stream_or_file: + The destination of the log messages. If None, stdout will be used. + :type stream_or_file: + `unicode` or `file` or None + :param debug: + Whether or not the logger will be at the DEBUG level (if False, the logger will be at the INFO level). + :type debug: + `bool` or None + :param name: + The logging channel. If None, a root logger will be created. + :type name: + `unicode` or None + :return: + A logger that's been set up according to the specified parameters. + :rtype: + :class:`Logger` + """ + logger = logging.getLogger(name) + if isinstance(stream_or_file, basestring): + handler = logging.FileHandler(stream_or_file, mode='w') + else: + handler = logging.StreamHandler(stream_or_file or sys.stdout) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG if debug else logging.INFO) + return logger diff --git a/requirements.txt b/requirements.txt index c8ca387f7..81e903184 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ cryptography>=0.9.2 +redis>=2.10.3 pyjwt>=1.3.0 requests>=2.4.3 six >= 1.4.0 diff --git a/setup.py b/setup.py index b51c97ad5..8e7e95a39 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ def run_tests(self): def main(): base_dir = dirname(__file__) install_requires = ['requests>=2.4.3', 'six>=1.4.0'] + redis_requires = ['redis>=2.10.3'] jwt_requires = ['pyjwt>=1.3.0', 'cryptography>=0.9.2'] if version_info < (3, 4): install_requires.append('enum34>=1.0.4') @@ -68,7 +69,7 @@ def main(): url='http://opensource.box.com', packages=find_packages(exclude=['demo', 'docs', 'test']), install_requires=install_requires, - extras_require={'jwt': jwt_requires}, + extras_require={'jwt': jwt_requires, 'redis': redis_requires}, tests_require=['pytest', 'pytest-xdist', 'mock', 'sqlalchemy', 'bottle', 'jsonpatch'], cmdclass={'test': PyTest}, classifiers=CLASSIFIERS, From e2135bbdb17747b7d087ef352a3cc8cc80e0e26a Mon Sep 17 00:00:00 2001 From: Jeffrey Meadows Date: Tue, 13 Oct 2015 18:06:32 -0700 Subject: [PATCH 2/5] Override _store_tokens instead of send_token_request in Redis Auth. --- boxsdk/auth/redis_managed_oauth2.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/boxsdk/auth/redis_managed_oauth2.py b/boxsdk/auth/redis_managed_oauth2.py index 921654941..3fa91f49e 100644 --- a/boxsdk/auth/redis_managed_oauth2.py +++ b/boxsdk/auth/redis_managed_oauth2.py @@ -48,14 +48,13 @@ def _get_tokens(self): self._update_current_tokens() return super(RedisManagedOAuth2Mixin, self)._get_tokens() - def send_token_request(self, *args, **kwargs): + def _store_tokens(self, access_token, refresh_token): """ Base class override. Saves the refreshed tokens in redis. """ - tokens = super(RedisManagedOAuth2Mixin, self).send_token_request(*args, **kwargs) - self._redis_server.hmset(self._unique_id, {'access': tokens[0], 'refresh': tokens[1]}) - return tokens + super(RedisManagedOAuth2Mixin, self)._store_tokens(access_token, refresh_token) + self._redis_server.hmset(self._unique_id, {'access': access_token, 'refresh': refresh_token}) class RedisManagedOAuth2(RedisManagedOAuth2Mixin): From 2bb05e38ea259cad7312373c3a3176580d45f2aa Mon Sep 17 00:00:00 2001 From: Jeffrey Meadows Date: Wed, 14 Oct 2015 13:53:22 -0700 Subject: [PATCH 3/5] Fix rst files and update docs. --- HISTORY.rst | 2 +- README.rst | 8 ++++---- docs/source/boxsdk.auth.rst | 16 ++++++++++++++++ docs/source/boxsdk.network.rst | 8 ++++++++ docs/source/boxsdk.util.rst | 8 ++++++++ 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 05e61201e..3457d1d48 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -12,7 +12,7 @@ Upcoming - Added a logging network class that logs requests and responses. - Added new options for auth classes, including storing tokens in Redis and storing them on a remote server. -1.2.1 (2015-07-22) +1.2.2 (2015-07-22) ++++++++++++++++++ - The SDK now supports setting a password when creating a shared link. diff --git a/README.rst b/README.rst index e7fecb95a..c9f51a00b 100644 --- a/README.rst +++ b/README.rst @@ -288,11 +288,11 @@ Other Auth Options For advanced uses of the SDK, two additional auth classes are provided: - `RemoteOAuth2`: Allows use of the SDK on clients without access to your application's client secret. Instead, you -provide a `retrieve_access_token` callback. That callback should perform the token refresh, perhaps on your server -that does have access to the client secret. + provide a `retrieve_access_token` callback. That callback should perform the token refresh, perhaps on your server + that does have access to the client secret. - `RedisManagedOAuth2`: Stores access and refresh tokens in Redis. This allows multiple processes (possibly spanning -multiple machines) to share access tokens while synchronizing token refresh. This could be useful for a multiprocess -web server, for example. + multiple machines) to share access tokens while synchronizing token refresh. This could be useful for a multiprocess + web server, for example. Other Network Options --------------------- diff --git a/docs/source/boxsdk.auth.rst b/docs/source/boxsdk.auth.rst index 67c41dc21..d17c1555d 100644 --- a/docs/source/boxsdk.auth.rst +++ b/docs/source/boxsdk.auth.rst @@ -20,6 +20,22 @@ boxsdk.auth.oauth2 module :undoc-members: :show-inheritance: +boxsdk.auth.redis_managed_oauth2 module +--------------------------------------- + +.. automodule:: boxsdk.auth.redis_managed_oauth2 + :members: + :undoc-members: + :show-inheritance: + +boxsdk.auth.remote_oauth2 module +-------------------------------- + +.. automodule:: boxsdk.auth.remote_oauth2 + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/source/boxsdk.network.rst b/docs/source/boxsdk.network.rst index 8f80a4f5b..3d0ee3859 100644 --- a/docs/source/boxsdk.network.rst +++ b/docs/source/boxsdk.network.rst @@ -12,6 +12,14 @@ boxsdk.network.default_network module :undoc-members: :show-inheritance: +boxsdk.network.logging_network module +------------------------------------- + +.. automodule:: boxsdk.network.logging_network + :members: + :undoc-members: + :show-inheritance: + boxsdk.network.network_interface module --------------------------------------- diff --git a/docs/source/boxsdk.util.rst b/docs/source/boxsdk.util.rst index abcec8923..562df4c27 100644 --- a/docs/source/boxsdk.util.rst +++ b/docs/source/boxsdk.util.rst @@ -12,6 +12,14 @@ boxsdk.util.compat module :undoc-members: :show-inheritance: +boxsdk.util.log module +---------------------- + +.. automodule:: boxsdk.util.log + :members: + :undoc-members: + :show-inheritance: + boxsdk.util.lru_cache module ---------------------------- From b35881fc1a8607ff6bf4d6ba4f9f11d96d240a84 Mon Sep 17 00:00:00 2001 From: Jeffrey Meadows Date: Mon, 2 Nov 2015 14:31:54 -0800 Subject: [PATCH 4/5] Add tests for new auth and logging classes. Add a new coop auth demo. --- README.rst | 1 + boxsdk/auth/cooperatively_managed_oauth2.py | 35 +++++++ boxsdk/auth/oauth2.py | 7 +- boxsdk/auth/redis_managed_oauth2.py | 7 +- ...ote_oauth2.py => remote_managed_oauth2.py} | 21 ++++- boxsdk/network/logging_network.py | 13 ++- demo/auth.py | 6 +- demo/cooperative_auth.py | 92 +++++++++++++++++++ demo/example.py | 2 +- demo/music_player.py | 2 +- docs/source/boxsdk.auth.rst | 14 ++- test/conftest.py | 1 + .../auth/test_cooperatively_managed_oauth2.py | 19 ++++ test/unit/auth/test_jwt_auth.py | 4 +- test/unit/auth/test_redis_managed_oauth2.py | 59 ++++++++++++ test/unit/auth/test_remote_managed_oauth2.py | 20 ++++ test/unit/network/conftest.py | 10 ++ test/unit/network/test_logging_network.py | 63 +++++++++++++ test/unit/network/test_network.py | 5 - 19 files changed, 354 insertions(+), 27 deletions(-) create mode 100644 boxsdk/auth/cooperatively_managed_oauth2.py rename boxsdk/auth/{remote_oauth2.py => remote_managed_oauth2.py} (55%) create mode 100644 demo/cooperative_auth.py create mode 100644 test/unit/auth/test_cooperatively_managed_oauth2.py create mode 100644 test/unit/auth/test_redis_managed_oauth2.py create mode 100644 test/unit/auth/test_remote_managed_oauth2.py create mode 100644 test/unit/network/conftest.py create mode 100644 test/unit/network/test_logging_network.py diff --git a/README.rst b/README.rst index c9f51a00b..5f6f6d32f 100644 --- a/README.rst +++ b/README.rst @@ -287,6 +287,7 @@ Other Auth Options For advanced uses of the SDK, two additional auth classes are provided: +- `CooperativelyManagedOAuth2`: Allows multiple auth instances to share tokens. - `RemoteOAuth2`: Allows use of the SDK on clients without access to your application's client secret. Instead, you provide a `retrieve_access_token` callback. That callback should perform the token refresh, perhaps on your server that does have access to the client secret. diff --git a/boxsdk/auth/cooperatively_managed_oauth2.py b/boxsdk/auth/cooperatively_managed_oauth2.py new file mode 100644 index 000000000..654ca3303 --- /dev/null +++ b/boxsdk/auth/cooperatively_managed_oauth2.py @@ -0,0 +1,35 @@ +# coding: utf-8 + +from __future__ import unicode_literals +from boxsdk import OAuth2 + + +class CooperativelyManagedOAuth2Mixin(OAuth2): + """ + Box SDK OAuth2 mixin. + Allows for sharing auth tokens between multiple clients. + """ + def __init__(self, retrieve_tokens=None, *args, **kwargs): + """ + :param retrieve_tokens: + Callback to get the current access/refresh token pair. + :type retrieve_tokens: + `callable` of () => (`unicode`, `unicode`) + """ + self._retrieve_tokens = retrieve_tokens + super(CooperativelyManagedOAuth2Mixin, self).__init__(*args, **kwargs) + + def _get_tokens(self): + """ + Base class override. Get the tokens from the user-specified callback. + """ + return self._retrieve_tokens() + + +class CooperativelyManagedOAuth2(CooperativelyManagedOAuth2Mixin): + """ + Box SDK OAuth2 subclass. + Allows for sharing auth tokens between multiple clients. The retrieve_tokens callback should + return the current access/refresh token pair. + """ + pass diff --git a/boxsdk/auth/oauth2.py b/boxsdk/auth/oauth2.py index 1cd3e95c9..bd9fea156 100644 --- a/boxsdk/auth/oauth2.py +++ b/boxsdk/auth/oauth2.py @@ -28,6 +28,7 @@ def __init__( access_token=None, refresh_token=None, network_layer=None, + refresh_lock=None, ): """ :param client_id: @@ -62,6 +63,10 @@ def __init__( If specified, use it to make network requests. If not, the default network implementation will be used. :type network_layer: :class:`Network` + :param refresh_lock: + Lock used to synchronize token refresh. If not specified, then a :class:`threading.Lock` will be used. + :type refresh_lock: + Context Manager """ self._client_id = client_id self._client_secret = client_secret @@ -69,7 +74,7 @@ def __init__( self._access_token = access_token self._refresh_token = refresh_token self._network_layer = network_layer if network_layer else DefaultNetwork() - self._refresh_lock = Lock() + self._refresh_lock = refresh_lock or Lock() self._box_device_id = box_device_id self._box_device_name = box_device_name diff --git a/boxsdk/auth/redis_managed_oauth2.py b/boxsdk/auth/redis_managed_oauth2.py index 3fa91f49e..0cdadb8d1 100644 --- a/boxsdk/auth/redis_managed_oauth2.py +++ b/boxsdk/auth/redis_managed_oauth2.py @@ -24,12 +24,15 @@ class RedisManagedOAuth2Mixin(OAuth2): def __init__(self, unique_id=uuid4(), redis_server=None, *args, **kwargs): self._unique_id = unique_id self._redis_server = redis_server or StrictRedis() - super(RedisManagedOAuth2Mixin, self).__init__(*args, **kwargs) + refresh_lock = Lock(redis=self._redis_server, name='{0}_lock'.format(self._unique_id)) + super(RedisManagedOAuth2Mixin, self).__init__(*args, refresh_lock=refresh_lock, **kwargs) if self._access_token is None: self._update_current_tokens() - self._refresh_lock = Lock(redis=self._redis_server, name='{0}_lock'.format(self._unique_id)) def _update_current_tokens(self): + """ + Get the latest tokens from redis and store them. + """ self._access_token, self._refresh_token = self._redis_server.hvals(self._unique_id) or (None, None) @property diff --git a/boxsdk/auth/remote_oauth2.py b/boxsdk/auth/remote_managed_oauth2.py similarity index 55% rename from boxsdk/auth/remote_oauth2.py rename to boxsdk/auth/remote_managed_oauth2.py index 5cf07e3e2..0436154cb 100644 --- a/boxsdk/auth/remote_oauth2.py +++ b/boxsdk/auth/remote_managed_oauth2.py @@ -6,15 +6,17 @@ class RemoteOAuth2Mixin(OAuth2): """ - Box SDK OAuth2 subclass. + Box SDK OAuth2 mixin. Allows for storing auth tokens remotely. - :param retrieve_access_token: - Callback to exchange an existing access token for a new one. - :type retrieve_access_token: - `callable` of `unicode` => `unicode` """ def __init__(self, retrieve_access_token=None, *args, **kwargs): + """ + :param retrieve_access_token: + Callback to exchange an existing access token for a new one. + :type retrieve_access_token: + `callable` of `unicode` => `unicode` + """ self._retrieve_access_token = retrieve_access_token super(RemoteOAuth2Mixin, self).__init__(*args, **kwargs) @@ -24,3 +26,12 @@ def _refresh(self, access_token): """ self._access_token = self._retrieve_access_token(access_token) return self._access_token, None + + +class RemoteOAuth2(RemoteOAuth2Mixin): + """ + Box SDK OAuth2 subclass. + Allows for storing auth tokens remotely. The retrieve_access_token callback should + return an access token, presumably acquired from a remote server on which your auth credentials are available. + """ + pass diff --git a/boxsdk/network/logging_network.py b/boxsdk/network/logging_network.py index 53799432e..3d29f2c8a 100644 --- a/boxsdk/network/logging_network.py +++ b/boxsdk/network/logging_network.py @@ -10,6 +10,11 @@ class LoggingNetwork(DefaultNetwork): """ SDK Network subclass that logs requests and responses. """ + LOGGER_NAME = 'boxsdk.network' + REQUEST_FORMAT = '\x1b[36m%s %s %s\x1b[0m' + SUCCESSFUL_RESPONSE_FORMAT = '\x1b[32m%s\x1b[0m' + ERROR_RESPONSE_FORMAT = '\x1b[31m%s\n%s\n%s\n\x1b[0m' + def __init__(self, logger=None): """ :param logger: @@ -19,7 +24,7 @@ def __init__(self, logger=None): :class:`Logger` """ super(LoggingNetwork, self).__init__() - self._logger = logger or setup_logging(name='network') + self._logger = logger or setup_logging(name=self.LOGGER_NAME) @property def logger(self): @@ -42,7 +47,7 @@ def _log_request(self, method, url, **kwargs): :type access_token: `unicode` """ - self._logger.info('\x1b[36m%s %s %s\x1b[0m', method, url, pformat(kwargs)) + self._logger.info(self.REQUEST_FORMAT, method, url, pformat(kwargs)) def _log_response(self, response): """ @@ -51,10 +56,10 @@ def _log_response(self, response): :param response: The Box API response. """ if response.ok: - self._logger.info('\x1b[32m%s\x1b[0m', response.content) + self._logger.info(self.SUCCESSFUL_RESPONSE_FORMAT, response.content) else: self._logger.warning( - '\x1b[31m%s\n%s\n%s\n\x1b[0m', + self.ERROR_RESPONSE_FORMAT, response.status_code, response.headers, pformat(response.content), diff --git a/demo/auth.py b/demo/auth.py index 994addb5a..92b0227c3 100644 --- a/demo/auth.py +++ b/demo/auth.py @@ -15,7 +15,7 @@ CLIENT_SECRET = '' # Insert Box client secret here -def authenticate(): +def authenticate(oauth_class=OAuth2): class StoppableWSGIServer(bottle.ServerAdapter): def __init__(self, *args, **kwargs): super(StoppableWSGIServer, self).__init__(*args, **kwargs) @@ -45,7 +45,7 @@ def get_token(): server_thread = Thread(target=lambda: local_oauth_redirect.run(server=local_server)) server_thread.start() - oauth = OAuth2( + oauth = oauth_class( client_id=CLIENT_ID, client_secret=CLIENT_SECRET, ) @@ -60,7 +60,7 @@ def get_token(): print('access_token: ' + access_token) print('refresh_token: ' + refresh_token) - return oauth + return oauth, access_token, refresh_token if __name__ == '__main__': diff --git a/demo/cooperative_auth.py b/demo/cooperative_auth.py new file mode 100644 index 000000000..fc0633313 --- /dev/null +++ b/demo/cooperative_auth.py @@ -0,0 +1,92 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from logging import Logger +from multiprocessing import Manager, Process +from os import getpid + +from boxsdk.auth.cooperatively_managed_oauth2 import CooperativelyManagedOAuth2 +from boxsdk.network.logging_network import LoggingNetwork +from boxsdk.util.log import setup_logging +from boxsdk import Client + +from auth import authenticate, CLIENT_ID, CLIENT_SECRET + + +def main(): + # Create a multiprocessing manager to use as the token store + global tokens, refresh_lock + manager = Manager() + tokens = manager.Namespace() + refresh_lock = manager.Lock() + + # Authenticate in master process + oauth2, tokens.access, tokens.refresh = authenticate(CooperativelyManagedOAuth2) + + # Create 2 worker processes and wait on them to finish + workers = [] + for _ in range(2): + worker_process = Process(target=worker) + worker_process.start() + workers.append(worker_process) + for worker_process in workers: + worker_process.join() + + +def _retrive_tokens(): + return tokens.access, tokens.refresh + + +def _store_tokens(access_token, refresh_token): + tokens.access, tokens.refresh = access_token, refresh_token + + +def worker(): + # Set up a logging network, but use the LoggingProxy so we can see which PID is generating messages + logger = setup_logging(name='boxsdk.network.{0}'.format(getpid())) + logger_proxy = LoggerProxy(logger) + logging_network = LoggingNetwork(logger) + + # Create a coop oauth2 instance. + # Tokens will be retrieved from and stored to the multiprocessing Namespace. + # A multiprocessing Lock will be used to synchronize token refresh. + # The tokens from the master process are used for initial auth. + # Whichever process needs to refresh + oauth2 = CooperativelyManagedOAuth2( + retrieve_tokens=_retrive_tokens, + client_id=CLIENT_ID, + client_secret=CLIENT_SECRET, + store_tokens=_store_tokens, + network_layer=logging_network, + access_token=tokens.access, + refresh_token=tokens.refresh, + refresh_lock=refresh_lock, + ) + client = Client(oauth2, network_layer=logging_network) + _do_work(client) + + +def _do_work(client): + # Do some work in a worker process. + # To see token refresh, perhaps put this in a loop (and don't forget to sleep for a bit between requests). + me = client.user(user_id='me').get() + items = client.folder('0').get_items(10) + + +class LoggerProxy(Logger): + """ + Proxy for a logger that injects the current PID before log messages. + """ + def __init__(self, logger): + self._logger_log = logger._log + logger._log = self._log + self._preamble = 'PID {0}: '.format(getpid()) + + def _log(self, level, msg, args, exc_info=None, extra=None): + msg = self._preamble + msg + return self._logger_log(level, msg, args, exc_info=exc_info, extra=extra) + + +if __name__ == '__main__': + main() diff --git a/demo/example.py b/demo/example.py index 630013137..9471ee675 100644 --- a/demo/example.py +++ b/demo/example.py @@ -290,7 +290,7 @@ def run_examples(oauth): def main(): # Please notice that you need to put in your client id and client secret in demo/auth.py in order to make this work. - oauth = authenticate() + oauth, _, _ = authenticate() run_examples(oauth) os._exit(0) diff --git a/demo/music_player.py b/demo/music_player.py index 368bf23c1..ee356192f 100644 --- a/demo/music_player.py +++ b/demo/music_player.py @@ -16,7 +16,7 @@ def __init__(self, folder_path): shuffle(self._mp3_files) def _get_client(self): - oauth = self._authenticate() + oauth, _, _ = self._authenticate() return Client(oauth) def _authenticate(self): diff --git a/docs/source/boxsdk.auth.rst b/docs/source/boxsdk.auth.rst index d17c1555d..7b85ad194 100644 --- a/docs/source/boxsdk.auth.rst +++ b/docs/source/boxsdk.auth.rst @@ -4,6 +4,14 @@ boxsdk.auth package Submodules ---------- +boxsdk.auth.cooperatively_managed_oauth2 module +----------------------------------------------- + +.. automodule:: boxsdk.auth.cooperatively_managed_oauth2 + :members: + :undoc-members: + :show-inheritance: + boxsdk.auth.jwt_auth module --------------------------- @@ -28,10 +36,10 @@ boxsdk.auth.redis_managed_oauth2 module :undoc-members: :show-inheritance: -boxsdk.auth.remote_oauth2 module --------------------------------- +boxsdk.auth.remote_managed_oauth2 module +---------------------------------------- -.. automodule:: boxsdk.auth.remote_oauth2 +.. automodule:: boxsdk.auth.remote_managed_oauth2 :members: :undoc-members: :show-inheritance: diff --git a/test/conftest.py b/test/conftest.py index 97ac30c18..abc506273 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -75,6 +75,7 @@ def retry_after_response(request): def server_error_response(request): mock_network_response = Mock(DefaultNetworkResponse) mock_network_response.status_code = int(request.param) + mock_network_response.ok = False return mock_network_response diff --git a/test/unit/auth/test_cooperatively_managed_oauth2.py b/test/unit/auth/test_cooperatively_managed_oauth2.py new file mode 100644 index 000000000..9a72f6f10 --- /dev/null +++ b/test/unit/auth/test_cooperatively_managed_oauth2.py @@ -0,0 +1,19 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from mock import Mock + +from boxsdk.auth import cooperatively_managed_oauth2 + + +def test_cooperatively_managed_oauth2_calls_retrieve_tokens_during_refresh(access_token, refresh_token): + retrieve_tokens = Mock() + oauth2 = cooperatively_managed_oauth2.CooperativelyManagedOAuth2( + retrieve_tokens=retrieve_tokens, + client_id=None, + client_secret=None, + ) + retrieve_tokens.return_value = access_token, refresh_token + assert oauth2.refresh(None) == (access_token, refresh_token) + retrieve_tokens.assert_called_once_with() diff --git a/test/unit/auth/test_jwt_auth.py b/test/unit/auth/test_jwt_auth.py index 621e736b0..3a5d8a713 100644 --- a/test/unit/auth/test_jwt_auth.py +++ b/test/unit/auth/test_jwt_auth.py @@ -79,9 +79,9 @@ def jwt_auth_init_mocks( ) jwt_auth_open.assert_called_once_with(sentinel.rsa_path) - key_file.return_value.read.assert_called_once_with() + key_file.return_value.read.assert_called_once_with() # pylint:disable=no-member load_pem_private_key.assert_called_once_with( - key_file.return_value.read.return_value, + key_file.return_value.read.return_value, # pylint:disable=no-member password=rsa_passphrase, backend=default_backend(), ) diff --git a/test/unit/auth/test_redis_managed_oauth2.py b/test/unit/auth/test_redis_managed_oauth2.py new file mode 100644 index 000000000..c03eec174 --- /dev/null +++ b/test/unit/auth/test_redis_managed_oauth2.py @@ -0,0 +1,59 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from mock import Mock, patch + +from boxsdk.auth import redis_managed_oauth2 + + +def test_redis_managed_oauth2_gets_tokens_from_redis_on_init(access_token, refresh_token): + redis_server = Mock(redis_managed_oauth2.StrictRedis) + redis_server.hvals.return_value = access_token, refresh_token + unique_id = Mock() + oauth2 = redis_managed_oauth2.RedisManagedOAuth2( + client_id=None, + client_secret=None, + unique_id=unique_id, + redis_server=redis_server, + ) + redis_server.hvals.assert_called_once_with(unique_id) + assert oauth2.unique_id is unique_id + + +def test_redis_managed_oauth2_gets_tokens_from_redis_during_refresh(access_token, refresh_token): + redis_server = Mock(redis_managed_oauth2.StrictRedis) + redis_server.hvals.return_value = access_token, refresh_token + unique_id = Mock() + with patch.object(redis_managed_oauth2.RedisManagedOAuth2Mixin, '_update_current_tokens'): + oauth2 = redis_managed_oauth2.RedisManagedOAuth2( + client_id=None, + client_secret=None, + unique_id=unique_id, + redis_server=redis_server, + ) + + assert oauth2.refresh('bogus_access_token') == (access_token, refresh_token) + redis_server.hvals.assert_called_once_with(unique_id) + + +def test_redis_managed_oauth2_stores_tokens_to_redis_during_refresh( + access_token, + refresh_token, + mock_network_layer, + successful_token_response, +): + redis_server = Mock(redis_managed_oauth2.StrictRedis) + redis_server.hvals.return_value = access_token, refresh_token + unique_id = Mock() + with patch.object(redis_managed_oauth2.RedisManagedOAuth2, '_update_current_tokens'): + oauth2 = redis_managed_oauth2.RedisManagedOAuth2( + client_id=None, + client_secret=None, + unique_id=unique_id, + redis_server=redis_server, + network_layer=mock_network_layer, + ) + mock_network_layer.request.return_value = successful_token_response + oauth2.send_token_request({}, access_token=None, expect_refresh_token=True) + redis_server.hmset.assert_called_once_with(unique_id, {'access': access_token, 'refresh': refresh_token}) diff --git a/test/unit/auth/test_remote_managed_oauth2.py b/test/unit/auth/test_remote_managed_oauth2.py new file mode 100644 index 000000000..b36066373 --- /dev/null +++ b/test/unit/auth/test_remote_managed_oauth2.py @@ -0,0 +1,20 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from mock import Mock + +from boxsdk.auth import remote_managed_oauth2 + + +def test_remote_managed_oauth2_calls_retrieve_tokens_during_refresh(access_token): + retrieve_access_token = Mock() + oauth2 = remote_managed_oauth2.RemoteOAuth2( + retrieve_access_token=retrieve_access_token, + client_id=None, + client_secret=None, + access_token=access_token, + ) + retrieve_access_token.return_value = access_token + assert oauth2.refresh(access_token) == (access_token, None) + retrieve_access_token.assert_called_once_with(access_token) diff --git a/test/unit/network/conftest.py b/test/unit/network/conftest.py new file mode 100644 index 000000000..b9e08f69d --- /dev/null +++ b/test/unit/network/conftest.py @@ -0,0 +1,10 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +import pytest + + +@pytest.fixture(params=('GET', 'POST', 'PUT', 'DELETE', 'OPTIONS')) +def http_verb(request): + return request.param diff --git a/test/unit/network/test_logging_network.py b/test/unit/network/test_logging_network.py new file mode 100644 index 000000000..7a56f54ae --- /dev/null +++ b/test/unit/network/test_logging_network.py @@ -0,0 +1,63 @@ +# coding: utf-8 + +from __future__ import unicode_literals, absolute_import + +from logging import Logger +from mock import Mock, patch + +from boxsdk.network import default_network, logging_network +from boxsdk.network.logging_network import LoggingNetwork + + +def test_logging_network_calls_setup_logging_if_logger_is_none(): + with patch.object(logging_network, 'setup_logging') as setup_logging: + network = LoggingNetwork() + setup_logging.assert_called_once_with(name=LoggingNetwork.LOGGER_NAME) + assert network.logger is setup_logging.return_value + + +def test_logging_network_does_not_call_setup_logging_if_logger_is_not_none(): + logger = Mock(Logger) + with patch.object(logging_network, 'setup_logging') as setup_logging: + network = LoggingNetwork(logger) + setup_logging.assert_never_called() + assert network.logger is logger + + +def test_logging_network_logs_requests(http_verb, test_url, access_token): + logger = Mock(Logger) + network = LoggingNetwork(logger) + with patch.object(logging_network, 'pformat') as pformat: + with patch.object(default_network.DefaultNetwork, 'request') as super_request: + network.request(http_verb, test_url, access_token, custom_kwarg='foo') + kwargs = pformat.return_value + super_request.assert_called_once_with(http_verb, test_url, access_token, custom_kwarg='foo') + pformat.assert_called_once_with(dict(custom_kwarg='foo')) + logger.info.assert_any_call(network.REQUEST_FORMAT, http_verb, test_url, kwargs) + + +def test_logging_network_logs_successful_responses(http_verb, test_url, access_token, generic_successful_response): + logger = Mock(Logger) + network = LoggingNetwork(logger) + with patch.object(default_network.DefaultNetwork, 'request') as super_request: + super_request.return_value = generic_successful_response + network.request(http_verb, test_url, access_token) + super_request.assert_called_once_with(http_verb, test_url, access_token) + logger.info.assert_called_with(network.SUCCESSFUL_RESPONSE_FORMAT, generic_successful_response.content) + + +def test_logging_network_logs_non_successful_responses(http_verb, test_url, access_token, server_error_response): + logger = Mock(Logger) + network = LoggingNetwork(logger) + with patch.object(logging_network, 'pformat') as pformat: + with patch.object(default_network.DefaultNetwork, 'request') as super_request: + super_request.return_value = server_error_response + network.request(http_verb, test_url, access_token) + super_request.assert_called_once_with(http_verb, test_url, access_token) + pformat.assert_called_with(server_error_response.content) + logger.warning.assert_called_once_with( + network.ERROR_RESPONSE_FORMAT, + server_error_response.status_code, + server_error_response.headers, + pformat.return_value, + ) diff --git a/test/unit/network/test_network.py b/test/unit/network/test_network.py index c71e6339f..8994e1784 100644 --- a/test/unit/network/test_network.py +++ b/test/unit/network/test_network.py @@ -15,11 +15,6 @@ def mock_request(monkeypatch): return session.request -@pytest.fixture(params=('GET', 'POST', 'PUT', 'DELETE', 'OPTIONS')) -def http_verb(request): - return request.param - - def test_default_network_response_properties_pass_through_to_session_response_properties(access_token): mock_session_response = Mock(Response) mock_session_response.status_code = 200 From 3fff5405544e36a73e5616a9e5a3a3ee8bbb67d3 Mon Sep 17 00:00:00 2001 From: Jeffrey Meadows Date: Thu, 5 Nov 2015 12:03:48 -0800 Subject: [PATCH 5/5] Fix pylint errors. --- test/functional/test_file_upload_update_download.py | 13 ------------- test/unit/util/test_multipart_stream.py | 1 + test/util/streamable_mock_open.py | 1 + 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/test/functional/test_file_upload_update_download.py b/test/functional/test_file_upload_update_download.py index dba056aec..3b8d2d475 100644 --- a/test/functional/test_file_upload_update_download.py +++ b/test/functional/test_file_upload_update_download.py @@ -45,16 +45,3 @@ def test_upload_then_download(box_client, test_file_path, test_file_content, fil expected_file_content = test_file_content.encode('utf-8') if isinstance(test_file_content, six.text_type)\ else test_file_content assert writeable_stream.getvalue() == expected_file_content - - -if __name__ == '__main__': - from test.functional.conftest import box_client, box_oauth, mock_box, Box - - class MonkeyPatch: - def setattr(self, target, attr, value): - setattr(target, attr, value) - - client_id, client_secret, login = 'client_id', 'client_secret', 'login' - box = mock_box(Box(), MonkeyPatch(), client_id, client_secret, 'user', login) - client = box_client(box_oauth(client_id, client_secret, login)) - test_upload_then_update(client, '/path/to/file', 'Hello', 'Goodbye', 'foo.txt') diff --git a/test/unit/util/test_multipart_stream.py b/test/unit/util/test_multipart_stream.py index bbff4f56f..b7fc195ce 100644 --- a/test/unit/util/test_multipart_stream.py +++ b/test/unit/util/test_multipart_stream.py @@ -18,6 +18,7 @@ def multipart_stream_files(request): def test_multipart_stream_orders_data_before_files(multipart_stream_data, multipart_stream_files): + # pylint:disable=redefined-outer-name if not multipart_stream_data and not multipart_stream_files: pytest.xfail('Encoder does not support empty fields.') stream = MultipartStream(multipart_stream_data, multipart_stream_files) diff --git a/test/util/streamable_mock_open.py b/test/util/streamable_mock_open.py index aec78fb5c..c1afc9232 100644 --- a/test/util/streamable_mock_open.py +++ b/test/util/streamable_mock_open.py @@ -23,6 +23,7 @@ def read(size=-1): handle.position += size return data + # pylint:disable=no-member handle.tell.side_effect = tell handle.len = len(read_data) handle.read.side_effect = read