diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..19e5ff4 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +include = + smartfile/*.py diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 0000000..12d5888 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1,2 @@ +repo_token: T89iPkB3rBdrSFoYL6v25QvtEUwNJnhuA +service_name: travis-ci diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..84b8b3c --- /dev/null +++ b/.gitignore @@ -0,0 +1,93 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +main.py + +/test/resources diff --git a/.travis.yml b/.travis.yml index d8aa259..e086024 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,18 @@ language: python python: - - "2.6" - "2.7" + - "3.3" + - "3.6" +before_install: + - sudo apt-get install librsync1 -qq install: - - pip install --timeout=30 pep8 --use-mirrors - - pip install --timeout=30 https://github.com/dcramer/pyflakes/tarball/master - - pip install --timeout=30 -r requirements.txt --use-mirrors - - pip install --timeout=30 -q -e . --use-mirrors + - pip install --timeout=30 pep8 + - pip install --timeout=30 pyflakes + - pip install --timeout=30 -r requirements.txt + - pip install --timeout=30 -q -e . before_script: - make verify script: - make test +after_success: + - coveralls diff --git a/MANIFEST.in b/MANIFEST.in index 9561fb1..8e5e8a9 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ +include requirements.txt include README.rst diff --git a/Makefile b/Makefile index ec392ec..b34d529 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,30 @@ +.PHONY: test test: - python tests.py + coverage run tests.py +.PHONY: verify verify: - pyflakes -x W smartfile - pep8 --exclude=migrations --ignore=E501,E225 smartfile + pyflakes smartfile + pep8 --ignore=E501,E225 smartfile +.PHONY: install install: python setup.py install +.PHONY: publish publish: python setup.py register python setup.py sdist upload + +.PHONY: profile +profile: + python profile.py + +.PHONY: clean +clean: + find . -name *.pyc -delete + +.PHONY: distclean +distclean: clean + rm -rf env + diff --git a/README.rst b/README.rst index 2b5ffaa..0f4c0ef 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,20 @@ -.. figure:: https://travis-ci.org/smartfile/client-python.png - :alt: Travis CI Status - :target: https://travis-ci.org/smartfile/client-python +.. image:: https://d2xtrvzo9unrru.cloudfront.net/brands/smartfile/logo.png + :alt: SmartFile A `SmartFile`_ Open Source project. `Read more`_ about how SmartFile uses and contributes to Open Source software. -.. figure:: http://www.smartfile.com/images/logo.jpg - :alt: SmartFile +.. image:: https://travis-ci.org/smartfile/client-python.png + :alt: Travis CI Status + :target: https://travis-ci.org/smartfile/client-python + +.. image:: https://coveralls.io/repos/smartfile/client-python/badge.png?branch=master + :target: https://coveralls.io/r/smartfile/client-python + :alt: Code Coverage + +.. image:: https://badge.fury.io/py/smartfile.svg + :target: https://badge.fury.io/py/smartfile + :alt: Latest PyPI version Summary ------------ @@ -120,7 +128,6 @@ Three methods are supported for providing API credentials using basic authentica >>> api = BasicClient(netrcfile='/etc/smartfile.keys') >>> api.get('/ping') - OAuth Authentication -------------------- @@ -131,7 +138,7 @@ Authentication using OAuth authentication is bit more complicated, as it involve >>> from smartfile import OAuthClient >>> api = OAuthClient('**********', '**********') >>> # Be sure to only call each method once for each OAuth login - >>> + >>> >>> # This is the first step with the client, which should be left alone >>> api.get_request_token() >>> # Redirect users to the following URL: @@ -188,18 +195,15 @@ File transfers Uploading and downloading files is supported. -To upload a file, pass either a file-like object or a tuple of -``(filename, file-like)`` as a kwarg. +To upload a file: .. code:: python - >>> from StringIO import StringIO - >>> data = StringIO('StringIO instance has no .name attribute!') >>> from smartfile import BasicClient >>> api = BasicClient() - >>> api.post('/path/data/', file=('foobar.png', data)) - >>> # Or use a file-like object with a name attribute - >>> api.post('/path/data/', file=file('foobar.png', 'rb')) + >>> file = open('test.txt', 'rb') + >>> api.upload('test.txt', file) + Downloading is automatic, if the ``'Content-Type'`` header indicates content other than the expected JSON return value, then a file-like object is @@ -207,26 +211,65 @@ returned. .. code:: python - >>> import shutil >>> from smartfile import BasicClient >>> api = BasicClient() - >>> f = api.get('/path/data/', 'foobar.png') - >>> with file('foobar.png', 'wb') as o: - >>> shutil.copyfileobj(f, o) + >>> api.download('foobar.png') + + +Tasks +----- Operations are long-running jobs that are not executed within the time frame of an API call. For such operations, a task is created, and the API can be used to poll the status of the task. +Move files + .. code:: python + >>> import logging >>> from smartfile import BasicClient + >>> >>> api = BasicClient() - >>> t = api.post('/path/oper/move/', src='/foobar.png', dst='/images/foobar.png') + >>> + >>> LOGGER = logging.getLogger(__name__) + >>> LOGGER.setLevel(logging.INFO) + >>> + >>> api.move('file.txt', '/newFolder') + >>> >>> while True: - >>> s = api.get('/task', t['uuid']) - >>> if s['status'] == 'SUCCESS': + >>> try: + >>> s = api.get('/task', api['uuid']) + >>> # Sleep to assure the user does not get rate limited + >>> time.sleep(1) + >>> if s['result']['status'] == 'SUCCESS': + >>> break + >>> elif s['result']['status'] == 'FAILURE': + >>> LOGGER.info("Task failure: " + s['uuid']) + >>> except Exception as e: + >>> print e >>> break + +Delete files + +.. code:: python + + >>> from smartfile import BasicClient + >>> api = BasicClient() + >>> api.remove('foobar.png') + .. _SmartFile: http://www.smartfile.com/ .. _Read more: http://www.smartfile.com/open-source.html + + + +Running Tests +-------------- +To run tests for the test.py file: +:: + nosetests -v tests.py + +To run tests for the test_smartfile.py file: +:: + API_KEY='****' API_PASSWORD='****' nosetests test diff --git a/profile.py b/profile.py new file mode 100644 index 0000000..7171309 --- /dev/null +++ b/profile.py @@ -0,0 +1,19 @@ +import os +import cProfile + +from StringIO import StringIO + +from smartfile import sync + + +s1 = StringIO(os.urandom(1024**2)) +s2 = StringIO(os.urandom(1024**2)) + +#blocks = sync.table(s1) +cProfile.run('blocks = sync.table(s1)') + +#ranges, blob = sync.delta(s2, blocks) +cProfile.run('ranges, blob = sync.delta(s2, blocks)') + +#out = sync.patch(s1, ranges, blob) +cProfile.run('out = sync.patch(s1, ranges, blob)') \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 3201f57..3faf34c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,7 @@ +six oauthlib requests requests_oauthlib +python-librsync +coveralls +coverage diff --git a/setup.py b/setup.py index b2f044c..597c561 100644 --- a/setup.py +++ b/setup.py @@ -2,18 +2,23 @@ import os import re -from distutils.core import setup +from setuptools import setup VERSION_PATTERN = re.compile(r'^[^#]*__version__\W*\=\W*["\'](.*)["\']') VERSION = None +with open('requirements.txt') as f: + required = f.read().splitlines() + +required = [r for r in required if not r.startswith('git')] + def get_path(path): return os.path.join(os.path.dirname(__file__), path) -with file(get_path('smartfile/__init__.py')) as f: - for line in f.xreadlines(): +with open(get_path('smartfile/__init__.py'), 'r') as f: + for line in f.readlines(): m = VERSION_PATTERN.search(line) if m: VERSION = m.group(1) @@ -25,25 +30,19 @@ def get_path(path): name = 'smartfile' -release = '1' -versrel = VERSION + '-' + release -long_description = file(get_path('README.rst')).read() +long_description = open(get_path('README.rst'), 'r').read() setup( name=name, - version=versrel, + version=VERSION, description='A Python client for the SmartFile API.', long_description=long_description, - requires=[ - 'oauthlib', - 'requests', - 'requests_oauthlib', - ], + install_requires=required, author='SmartFile', - author_email='info@smartfile.com', + author_email='tech@smartfile.com', maintainer='Ben Timby', - maintainer_email='btimby@gmail.com', + maintainer_email='tech@smartfile.com', url='http://github.com/smartfile/client-python/', license='MIT', packages=['smartfile'], diff --git a/smartfile/__init__.py b/smartfile/__init__.py index fa7a3a6..fe64074 100644 --- a/smartfile/__init__.py +++ b/smartfile/__init__.py @@ -1,13 +1,18 @@ -import re import os +import re +import shutil import time -import string import urllib -import urlparse -import requests from netrc import netrc +try: + import urlparse + # Fixed pyflakes warning... + urlparse +except ImportError: + from urllib import parse as urlparse +import requests from requests.exceptions import RequestException from smartfile.errors import APIError @@ -15,7 +20,8 @@ from smartfile.errors import ResponseError -__version__ = '2.1' +__version__ = '2.19' +__major__ = __version__.split('.')[0] API_URL = 'https://app.smartfile.com/' @@ -24,19 +30,19 @@ def clean_tokens(*args): - args = map(string.strip, args) + if not all(map(bool, args)): + raise ValueError("not provided") + args = list(map(lambda x: x.strip(), args)) for i, arg in enumerate(args): if len(arg) < 30: - raise ValueError("Too short") - if not isinstance(arg, unicode): - arg = unicode(arg) + raise ValueError("too short") args[i] = arg return args class Client(object): """Base API client, handles communication, retry, versioning etc.""" - def __init__(self, url=None, version=__version__, throttle_wait=True): + def __init__(self, url=None, version=__major__, throttle_wait=True): self.url = url or os.environ.get('SMARTFILE_API_URL') or API_URL self.version = version self.throttle_wait = throttle_wait @@ -45,22 +51,26 @@ def _do_request(self, request, url, **kwargs): "Actually makes the HTTP request." try: response = request(url, stream=True, **kwargs) - except RequestException, e: + except RequestException as e: raise RequestError(e) else: if response.status_code >= 400: raise ResponseError(response) - # Try to return the response in the most useful fashion given it's type. + # Try to return the response in the most useful fashion given it's + # type. if response.headers.get('content-type') == 'application/json': try: # Try to decode as JSON return response.json() - except ValueError: + except (TypeError, ValueError): # If that fails, return the text. return response.text else: # This might be a file, so return it. - return response.raw + if kwargs.get('params', {}).get('raw', True): + return response.raw + else: + return response def _request(self, method, endpoint, id=None, **kwargs): "Handles retrying failed requests and error handling." @@ -71,7 +81,7 @@ def _request(self, method, endpoint, id=None, **kwargs): data = kwargs.get('data') if data: files = {} - for name, value in data.items(): + for name, value in list(data.items()): # Value might be a file-like object (with a read method), or it # might be a (filename, file-like) tuple. if hasattr(value, 'read') or isinstance(value, tuple): @@ -90,18 +100,21 @@ def _request(self, method, endpoint, id=None, **kwargs): path = path.replace('//', '/') url = self.url + path # Add our user agent. - kwargs.setdefault('headers', {}).setdefault('User-Agent', HTTP_USER_AGENT) + kwargs.setdefault('headers', {}).setdefault('User-Agent', + HTTP_USER_AGENT) # Now try the request, if we get throttled, sleep and try again. trys, retrys = 0, 3 while True: if trys == retrys: - raise RequestError('Could not complete request after %s trys.' % trys) + raise RequestError('Could not complete request after %s trys.' + % trys) trys += 1 try: return self._do_request(request, url, **kwargs) - except ResponseError, e: + except ResponseError as e: if self.throttle_wait and e.status_code == 503: - m = THROTTLE_PATTERN.match(e.response.headers.get('x-throttle', '')) + m = THROTTLE_PATTERN.match( + e.response.headers.get('x-throttle', '')) if m: time.sleep(float(m.group(1))) continue @@ -123,6 +136,48 @@ def post(self, endpoint, id=None, **kwargs): def delete(self, endpoint, id=None, **kwargs): return self._request('delete', endpoint, id=id, data=kwargs) + def remove(self, deletefile): + try: + return self.post('/path/oper/remove', path=deletefile) + except KeyError: + raise Exception("Destination file does not exist") + + def upload(self, filename, fileobj): + if filename.endswith('/'): + filename = filename[:-1] + arg = (filename, fileobj) + return self.post('/path/data/', file=arg) + + def download(self, file_to_be_downloaded, perform_download=True, download_to_path=None): + """ file_to_be_downloaded is a file-like object that has already + been uploaded, you cannot download folders """ + response = self.get( + '/path/data/', file_to_be_downloaded, raw=False) + if not perform_download: + # The caller can decide how to process the download of the data + return response + if not download_to_path: + download_to_path = file_to_be_downloaded.split("/")[-1] + # download uses shutil.copyfileobj to download, which copies + # the data in chunks + o = open(download_to_path, 'wb') + return shutil.copyfileobj(response.raw, o) + + def move(self, src_path, dst_path): + # check destination folder for / at end + if not src_path.endswith("/"): + src_path = src_path + "/" + # check destination folder for / at begining + if not src_path.startswith("/"): + src_path = "/" + src_path + # check destination folder for / at end + if not dst_path.endswith("/"): + dst_path = dst_path + "/" + # check destination folder for / at begining + if not dst_path.startswith("/"): + dst_path = "/" + dst_path + return self.post('/path/oper/move/', src=src_path, dst=dst_path) + class BasicClient(Client): """API client that uses a key and password. Layers a simple form of @@ -165,15 +220,13 @@ def _do_request(self, *args, **kwargs): from requests_oauthlib import OAuth1 from oauthlib.oauth1 import SIGNATURE_PLAINTEXT - #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~ - # OAuth, if available. - #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~ + # OAuth, if available. class OAuthToken(object): "Internal representation of an OAuth (token, secret) tuple." def __init__(self, token=None, secret=None): - self.token = token and unicode(token) - self.secret = secret and unicode(secret) + self.token = token + self.secret = secret def __iter__(self): yield self.token @@ -191,10 +244,11 @@ def is_valid(self): return False class OAuthClient(Client): - """API client that uses OAuth tokens. Layers a more complex form of - authentication useful for 3rd party access on top of the base Client.""" - def __init__(self, client_token=None, client_secret=None, access_token=None, - access_secret=None, **kwargs): + """API client that uses OAuth tokens. Layers a more complex + form of authentication useful for 3rd party access on top of + the base Client.""" + def __init__(self, client_token=None, client_secret=None, + access_token=None, access_secret=None, **kwargs): if client_token is None: client_token = os.environ.get('SMARTFILE_CLIENT_TOKEN') if client_secret is None: @@ -205,15 +259,15 @@ def __init__(self, client_token=None, client_secret=None, access_token=None, access_secret = os.environ.get('SMARTFILE_ACCESS_SECRET') self._client = OAuthToken(client_token, client_secret) if not self._client.is_valid(): - raise APIError('You must provide a client_token and client_secret ' - 'for OAuth.') + raise APIError('You must provide a client_token' + 'and client_secret for OAuth.') self._access = OAuthToken(access_token, access_secret) super(OAuthClient, self).__init__(**kwargs) def _do_request(self, *args, **kwargs): if not self._access.is_valid(): - raise APIError('You must obtain an access token before making API ' - 'calls.') + raise APIError('You must obtain an access token' + 'before making API calls.') # Add the OAuth parameters. kwargs['auth'] = OAuth1(self._client.token, client_secret=self._client.secret, @@ -224,34 +278,33 @@ def _do_request(self, *args, **kwargs): def get_request_token(self, callback=None): "The first step of the OAuth workflow." - if callback: - callback = unicode(callback) oauth = OAuth1(self._client.token, client_secret=self._client.secret, callback_uri=callback, signature_method=SIGNATURE_PLAINTEXT) - r = requests.post(urlparse.urljoin(self.url, 'oauth/request_token/'), auth=oauth) + r = requests.post(urlparse.urljoin( + self.url, 'oauth/request_token/'), auth=oauth) credentials = urlparse.parse_qs(r.text) self.__request = OAuthToken(credentials.get('oauth_token')[0], - credentials.get('oauth_token_secret')[0]) + credentials.get( + 'oauth_token_secret')[0]) return self.__request def get_authorization_url(self, request=None): "The second step of the OAuth workflow." if request is None: if not self.__request.is_valid(): - raise APIError('You must obtain a request token to request ' - 'and access token. Use get_request_token() ' - 'first.') + raise APIError('You must obtain a request token to' + 'request and access token. Use' + 'get_request_token() first.') request = self.__request url = urlparse.urljoin(self.url, 'oauth/authorize/') - return url + '?' + urllib.urlencode(dict(oauth_token=request.token)) + return url + '?' + urllib.urlencode( + dict(oauth_token=request.token)) def get_access_token(self, request=None, verifier=None): - """The final step of the OAuth workflow. After this the client can make - API calls.""" - if verifier: - verifier = unicode(verifier) + """The final step of the OAuth workflow. After this the client + can make API calls.""" if request is None: if not self.__request.is_valid(): raise APIError('You must obtain a request token to request ' @@ -262,9 +315,10 @@ def get_access_token(self, request=None, verifier=None): client_secret=self._client.secret, resource_owner_key=request.token, resource_owner_secret=request.secret, - verifier=unicode(verifier), + verifier=verifier, signature_method=SIGNATURE_PLAINTEXT) - r = requests.post(urlparse.urljoin(self.url, 'oauth/access_token/'), auth=oauth) + r = requests.post(urlparse.urljoin( + self.url, 'oauth/access_token/'), auth=oauth) credentials = urlparse.parse_qs(r.text) self._access = OAuthToken(credentials.get('oauth_token')[0], credentials.get('oauth_token_secret')[0]) @@ -272,9 +326,7 @@ def get_access_token(self, request=None, verifier=None): except ImportError: - #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~ - # OAuth, if not available. - #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~ + # OAuth, if not available. # Instead of a class, define this as a function, thus when a user tries to # "instantiate" it, they receive an exception. diff --git a/smartfile/errors.py b/smartfile/errors.py index f6aea4e..c741929 100644 --- a/smartfile/errors.py +++ b/smartfile/errors.py @@ -1,3 +1,4 @@ +import six class APIError(Exception): @@ -25,14 +26,22 @@ def __init__(self, response, *args, **kwargs): json = response.json() except ValueError: if self.status_code == 404: - self.detail = u'Invalid URL, check your API path' + self.detail = six.u('Invalid URL, check your API path') else: - self.detail = u'Server error; check response for errors' + self.detail = six.u('Server error; check response for errors') else: if self.status_code == 400 and 'field_errors' in json: self.detail = json['field_errors'] else: - self.detail = json['detail'] + try: + # A faulty move request returns the below response + self.detail = json['src'][0] + except KeyError: + # A faulty delete request returns the below response + try: + self.detail = json['path'][0] + except KeyError: + self.detail = six.u('Error: %s' % response.content) super(ResponseError, self).__init__(*args, **kwargs) def __str__(self): diff --git a/smartfile/sync.py b/smartfile/sync.py new file mode 100644 index 0000000..861711a --- /dev/null +++ b/smartfile/sync.py @@ -0,0 +1,121 @@ +import os +import errno +import tempfile + +try: + import librsync +except ImportError: + raise ImportError('python-librsync is required for sync capabilities. ' + 'Install it using `pip install python-librsync`.') + + +class BaseFile(object): + """ + Base class for files being synchronized. + """ + def __init__(self, path): + self.path = path + + +class LocalFile(BaseFile): + """ + Represents a local file that is being synchronized. Uses librsync to + perform the steps of the rsync algorithm. + """ + def signature(self, block_size=None): + "Calculates signature for local file." + kwargs = {} + if block_size: + kwargs['block_size'] = block_size + return librsync.signature(open(self.path, 'rb'), **kwargs) + + def delta(self, signature): + "Generates delta for local file using remote signature." + return librsync.delta(open(self.path, 'rb'), signature) + + def patch(self, delta): + "Applies remote delta to local file." + # Create a temp file in which to store our synced copy. We will handle + # deleting it manually, since we may move it instead. + with (tempfile.NamedTemporaryFile(prefix='.sync', + suffix=os.path.basename(self.path), + dir=os.path.dirname(self.path), delete=False)) as output: + try: + # Open the local file, data may be read from it. + with open(self.path, 'rb') as reference: + # Patch the local file into our temporary file. + r = librsync.patch(reference, delta, output) + os.rename(output.name, self.path) + return r + finally: + try: + os.remove(output.name) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + +class RemoteFile(BaseFile): + """ + Represents a remote file that is being synchronized. Makes API calls to + perform the steps of the rsync algorithm. + """ + def __init__(self, path, api): + super(RemoteFile, self).__init__(path) + self.api = api + + def signature(self, block_size=None): + "Requests a signature for remote file via API." + kwargs = {} + if block_size: + kwargs['block_size'] = block_size + return self.api.get('path/sync/signature', self.path, **kwargs) + + def delta(self, signature): + "Generates delta for remote file via API using local file's signature." + return self.api.post('path/sync/delta', self.path, signature=signature) + + def patch(self, delta): + "Applies delta for local file to remote file via API." + return self.api.post('path/sync/patch', self.path, delta=delta) + + +class SyncClient(object): + """ + Synchronizes remote and local files. + """ + def __init__(self, api, block_size=None): + """ + Synchronizes files with SmartFile using the sync API. + """ + self.api = api + self.block_size = block_size + + @property + def version(self): + return self.api.version + + def sync(self, src, dst): + """ + Performs synchronization from source to destination. Performs the three + steps: + + 1. Calculate signature of destination. + 2. Generate delta from source. + 3. Apply delta to destination. + """ + return dst.patch(src.delta(dst.signature(block_size=self.block_size))) + + def upload(self, local, remote): + """ + Performs synchronization from a local file to a remote file. The local + path is the source and remote path is the destination. + """ + self.sync(LocalFile(local), RemoteFile(remote, self.api)) + + def download(self, local, remote): + """ + Performs synchronization from a remote file to a local file. The + remote path is the source and the local path is the destination. + """ + self.sync(RemoteFile(remote, self.api), LocalFile(local)) diff --git a/test/test_smartfile.py b/test/test_smartfile.py new file mode 100644 index 0000000..247de6c --- /dev/null +++ b/test/test_smartfile.py @@ -0,0 +1,68 @@ +from __future__ import absolute_import + +import os +import requests +import unittest + +try: + from StringIO import StringIO +except ImportError: + from io import StringIO + +from smartfile import BasicClient +from smartfile.errors import ResponseError + +API_KEY = os.environ.get("API_KEY") +API_PASSWORD = os.environ.get("API_PASSWORD") + +if API_KEY is None: + raise RuntimeError("API_KEY is required") + +if API_PASSWORD is None: + raise RuntimeError("API_PASSWORD is required") + +TESTFN = "testfn" +file_contents = "hello" +TESTFN2 = "testfn2" + + +class CustomOperationsTestCase(unittest.TestCase): + + def setUp(self): + self.api = BasicClient(API_KEY, API_PASSWORD) + # Make directory for tests + self.api.post('/path/oper/mkdir/', path=TESTFN2) + + def get_data(self): + data = self.api.get("/path/info/testfn") + return data + + def tearDown(self): + self.api.remove('/testfn2') + os.remove('testfn') + + def upload(self): + f = StringIO(file_contents) + f.seek(0) + self.api.upload(TESTFN, f) + self.assertEquals(self.get_data()['size'], f.tell()) + + def download(self): + response = self.api.download(TESTFN, False) + self.assertTrue(isinstance(response, requests.Response)) + self.api.download(TESTFN) + self.assertEquals(self.get_data()['size'], os.path.getsize(TESTFN)) + + def move(self): + self.api.move(TESTFN, TESTFN2) + + def remove(self): + self.api.remove(os.path.join(TESTFN2, TESTFN)) + with self.assertRaises(ResponseError): + self.api.remove(os.path.join(TESTFN2, TESTFN)) + + def test_upload_download_move_delete(self): + self.upload() + self.download() + self.move() + self.remove() diff --git a/tests.py b/tests.py index 1dfa800..23e5297 100644 --- a/tests.py +++ b/tests.py @@ -1,14 +1,20 @@ -# -*- coding: utf-8 -*- - -import os +import cgi import json -import urlparse -import unittest +import os import tempfile import threading - -from BaseHTTPServer import HTTPServer -from BaseHTTPServer import BaseHTTPRequestHandler +import unittest +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + +try: + from BaseHTTPServer import HTTPServer + from BaseHTTPServer import BaseHTTPRequestHandler +except ImportError: + from http.server import HTTPServer + from http.server import BaseHTTPRequestHandler from smartfile import BasicClient from smartfile import OAuthClient @@ -29,34 +35,49 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler): A simple handler that logs requests for examination. """ class TestRequest(object): - def __init__(self, method, path, query=None, data=None): - self.method = method + def __init__(self, method, path, query=None, data=None, headers=None): + self.method = method.upper() self.path = path self.query = query self.data = data + self.headers = headers def __init__(self, *args, **kwargs): self.verbose = kwargs.pop('verbose', False) BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def record(self, method, path, query=None, data=None): - self.server.requests.append(TestHTTPRequestHandler.TestRequest(method, - path, query=query, data=data)) - - def respond(self): + request = TestHTTPRequestHandler.TestRequest( + method, + path, + query=query, + data=data, + headers=dict( + self.headers.items() + ) + ) + self.server.requests.append(request) + return request + + def respond(self, request): self.send_response(200) self.send_header("Content-type", "text/plain") self.end_headers() - self.wfile.write("Hello World!") + self.wfile.write(b"Hello World!") def parse_and_record(self, method): urlp = urlparse.urlparse(self.path) query, data = urlparse.parse_qs(urlp.query), None - if method == 'POST': + if method in ('POST', 'PUT'): l = int(self.headers['Content-Length']) - data = urlparse.parse_qs(self.rfile.read(l)) - self.record(method, urlp.path, query=query, data=data) - self.respond() + ct, params = cgi.parse_header(self.headers['Content-Type']) + if ct == 'multipart/form-data': + data = cgi.FieldStorage(fp=self.rfile, headers=self.headers, + environ={'REQUEST_METHOD': 'POST'}) + else: + data = urlparse.parse_qs(self.rfile.read(l)) + request = self.record(method, urlp.path, query=query, data=data) + self.respond(request) def log_message(self, *args, **kwargs): if self.verbose: @@ -82,7 +103,7 @@ class TestHTTPServer(threading.Thread, HTTPServer): """ allow_reuse_address = True - def __init__(self, address='127.0.0.1', port=0, handler=TestHTTPRequestHandler): + def __init__(self, handler, address='127.0.0.1', port=0): HTTPServer.__init__(self, (address, port), handler) threading.Thread.__init__(self) self.requests = [] @@ -97,8 +118,10 @@ class TestServerTestCase(unittest.TestCase): """ Test case that starts our test HTTP server. """ + handler = TestHTTPRequestHandler + def setUp(self): - self.server = TestHTTPServer() + self.server = TestHTTPServer(self.handler) def tearDown(self): self.server.shutdown() @@ -111,26 +134,39 @@ def assertRequestCount(self, num=1): elif requests < num: raise AssertionError('Less than %s request performed' % num) - def assertMethod(self, method): + def assertMethod(self, method, request=-1): try: - request = self.server.requests[0] + request = self.server.requests[request] except IndexError: raise AssertionError('Cannot assert method without request') - if request.method != method: - raise AssertionError('%s is not %s method' % (method, - request.method)) + self.assertEqual(method.upper(), request.method) - def assertPath(self, path): + def assertPath(self, path, request=-1): try: - request = self.server.requests[0] + request = self.server.requests[request] except IndexError: raise AssertionError('Cannot assert path without request') - if request.path != path: - raise AssertionError('"%s" is not equal to "%s"' % (path, - request.path)) + self.assertEqual(path, request.path) + + def assertData(self, key, value, request=-1): + try: + request = self.server.requests[request] + except IndexError: + raise AssertionError('Cannot assert data without request') + self.assertIn(value, request.data.getvalue(key, [])) + + def assertIn(self, test_value, expected_set): + msg = "%s did not occur in %s" % (test_value, expected_set) + self.assert_(test_value in expected_set, msg) + + +class ClientTestCase(TestServerTestCase): + def setUp(self): + super(ClientTestCase, self).setUp() + self.client = self.getClient() -class BasicTestCase(TestServerTestCase): +class BasicTestCase(ClientTestCase): def getClient(self, **kwargs): kwargs.setdefault('key', API_KEY) kwargs.setdefault('password', API_PASSWORD) @@ -139,7 +175,7 @@ def getClient(self, **kwargs): return BasicClient(**kwargs) -class OAuthTestCase(TestServerTestCase): +class OAuthTestCase(ClientTestCase): def getClient(self, **kwargs): kwargs.setdefault('client_token', CLIENT_TOKEN) kwargs.setdefault('client_secret', CLIENT_SECRET) @@ -153,106 +189,92 @@ def getClient(self, **kwargs): class UrlGenerationTestCase(object): "Tests that validate 'auto-generated' URLs." def test_with_path_id(self): - client = self.getClient() - client.get('/path/data', '/the/file/path') + self.client.get('/path/data', '/the/file/path') self.assertMethod('GET') self.assertPath('/api/{0}/path/data/the/file/path/'.format( - client.version)) + self.client.version)) def test_with_int_id(self): - client = self.getClient() - client.get('/access/user', 42) + self.client.get('/access/user', 42) self.assertMethod('GET') - self.assertPath('/api/{0}/access/user/42/'.format(client.version)) + self.assertPath('/api/{0}/access/user/42/'.format(self.client.version)) def test_with_version(self): - client = self.getClient(version='3.1') - client.get('/ping') - self.assertMethod('GET') - self.assertPath('/api/{0}/ping/'.format(client.version)) + for major in range(10): + for minor in range(10): + client = self.getClient(version='%s.%s' % (major, minor)) + client.get('/ping') + self.assertMethod('GET') + self.assertPath('/api/{0}/ping/'.format(client.version)) class MethodTestCase(object): "Tests the HTTP methods used by CRUD methods." def test_call_is_GET(self): - client = self.getClient() - client('/user', 'bobafett') + self.client('/user', 'bobafett') self.assertMethod('GET') def test_post_is_POST(self): - client = self.getClient() - client.post('/user', username='bobafett', email='bobafett@example.com') + self.client.post('/user', username='bobafett', + email='bobafett@example.com') self.assertMethod('POST') def test_get_is_GET(self): - client = self.getClient() - client.get('/user', 'bobafett') + self.client.get('/user', 'bobafett') self.assertMethod('GET') def test_put_is_PUT(self): - client = self.getClient() - client.put('/user', 'bobafett', full_name='Boba Fett') + self.client.put('/user', 'bobafett', full_name='Boba Fett') self.assertMethod('PUT') def test_delete_is_DELETE(self): - client = self.getClient() - client.delete('/user', 'bobafett') + self.client.delete('/user', 'bobafett') self.assertMethod('DELETE') class DownloadTestCase(object): def test_file_response(self): - client = self.getClient() - r = client.get('/user') + r = self.client.get('/user') self.assertTrue(hasattr(r, 'read'), 'File-like object not returned.') - self.assertEqual(r.read(), 'Hello World!') + self.assertEqual(r.read(), b'Hello World!') class UploadTestCase(object): def test_file_upload(self): - client = self.getClient() fd, t = tempfile.mkstemp() os.close(fd) try: - client.post('/path/data', 'foobar.png', file=file(t)) - except Exception, e: - self.fail('POSTing a file failed. %s' % e) + self.client.post('/path/data', 'foobar.png', file=open(t, 'rb')) finally: - try: - os.unlink(t) - except: - pass + os.unlink(t) -class BasicEnvironTestCase(BasicTestCase): +class BasicEnvironTestCase(UrlGenerationTestCase, BasicTestCase): "Tests that the API client reads settings from ENV." def setUp(self): - super(BasicEnvironTestCase, self).setUp() os.environ['SMARTFILE_API_KEY'] = API_KEY os.environ['SMARTFILE_API_PASSWORD'] = API_KEY + super(BasicEnvironTestCase, self).setUp() def tearDown(self): super(BasicEnvironTestCase, self).tearDown() del os.environ['SMARTFILE_API_KEY'] del os.environ['SMARTFILE_API_PASSWORD'] - def test_read_from_env(self): - # Blank out the credentials, the client should read them from the - # environment variables. - client = self.getClient(key=None, password=None) - client.get('/ping') - self.assertMethod('GET') - self.assertPath('/api/{0}/ping/'.format(client.version)) + def getClient(self, **kwargs): + kwargs['key'] = None + kwargs['password'] = None + return super(BasicEnvironTestCase, self).getClient(**kwargs) -class OAuthEnvironTestCase(OAuthTestCase): +class OAuthEnvironTestCase(UrlGenerationTestCase, OAuthTestCase): "Tests that the API client reads settings from ENV." def setUp(self): - super(OAuthEnvironTestCase, self).setUp() os.environ['SMARTFILE_CLIENT_TOKEN'] = CLIENT_TOKEN os.environ['SMARTFILE_CLIENT_SECRET'] = CLIENT_SECRET os.environ['SMARTFILE_ACCESS_TOKEN'] = ACCESS_TOKEN os.environ['SMARTFILE_ACCESS_SECRET'] = ACCESS_SECRET + super(OAuthEnvironTestCase, self).setUp() def tearDown(self): super(OAuthEnvironTestCase, self).tearDown() @@ -261,13 +283,10 @@ def tearDown(self): del os.environ['SMARTFILE_ACCESS_TOKEN'] del os.environ['SMARTFILE_ACCESS_SECRET'] - def test_read_from_env(self): - # Blank out the credentials, the client should read them from the - # environment variables. - client = self.getClient(client_token=None, client_secret=None) - client.get('/ping') - self.assertMethod('GET') - self.assertPath('/api/{0}/ping/'.format(client.version)) + def getClient(self, **kwargs): + kwargs['client_token'] = None + kwargs['client_secret'] = None + return super(OAuthEnvironTestCase, self).getClient(**kwargs) class BasicClientTestCase(DownloadTestCase, UploadTestCase, MethodTestCase, @@ -284,8 +303,9 @@ def test_netrc(self): address, port = address else: port = self.server.server_port - netrc = "machine 127.0.0.1:%s\n login %s\n password %s" % ( + netrc = 'machine 127.0.0.1:%i\n login %s\n password %s' % ( port, API_KEY, API_PASSWORD) + netrc = netrc.encode('utf8') os.write(fd, netrc) finally: os.close(fd) @@ -303,7 +323,8 @@ def test_netrc(self): class OAuthClientTestCase(DownloadTestCase, UploadTestCase, MethodTestCase, UrlGenerationTestCase, OAuthTestCase): def test_blank_client_token(self): - self.assertRaises(APIError, self.getClient, client_token='', client_secret='') + self.assertRaises(APIError, self.getClient, + client_token='', client_secret='') def test_blank_access_token(self): client = self.getClient(access_token='', access_secret='') @@ -311,20 +332,18 @@ def test_blank_access_token(self): class HTTPThrottleRequestHandler(TestHTTPRequestHandler): - def respond(self): + def respond(self, request): self.send_response(503) self.send_header("X-Throttle", "throttled; next=0.01 sec") self.end_headers() - self.wfile.write("Request Throttled!") + self.wfile.write(b"Request Throttled!") class ThrottleTestCase(object): - def setUp(self): - self.server = TestHTTPServer(handler=HTTPThrottleRequestHandler) + handler = HTTPThrottleRequestHandler def test_throttle_GET(self): - client = self.getClient() - self.assertRaises(RequestError, client.get, '/ping') + self.assertRaises(RequestError, self.client.get, '/ping') self.assertRequestCount(3) @@ -337,22 +356,20 @@ class OAuthThrottleTestCase(ThrottleTestCase, OAuthTestCase): class HTTPJSONRequestHandler(TestHTTPRequestHandler): - def respond(self): + def respond(self, request): self.send_response(200) self.send_header("Content-Type", "application/json") self.end_headers() - self.wfile.write(json.dumps({ 'foo': 'bar' })) + self.wfile.write(json.dumps({'foo': 'bar'}).encode('utf8')) class JSONTestCase(object): - def setUp(self): - self.server = TestHTTPServer(handler=HTTPJSONRequestHandler) + handler = HTTPJSONRequestHandler def test_throttle_GET(self): - client = self.getClient() - r = client.get('/user') + r = self.client.get('/user') self.assertMethod('GET') - self.assertEqual(r, { 'foo': 'bar' }) + self.assertEqual(r, {'foo': 'bar'}) class BasicJSONTestCase(JSONTestCase, BasicTestCase):