From 145c3c4f5bbafcfc7d8d97caec2eba8c50183e5b Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 30 Apr 2013 23:17:18 -0400
Subject: [PATCH 01/90] Added simple sync algorithm implementation. This will
 compliment coming sync API. Unit tests to ensure all parts of this
 implementation function. Coming next will be the sync API client and a
 high-level sync implementation.

---
 smartfile/__init__.py |   4 +-
 smartfile/sync.py     | 171 +++++++++++++++++++++++++++++++++
 tests.py              | 215 +++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 388 insertions(+), 2 deletions(-)
 create mode 100644 smartfile/sync.py

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 48897dc..a83aac4 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -24,10 +24,12 @@
 
 
 def clean_tokens(*args):
+    if not all(map(bool, args)):
+        raise ValueError("not provided")
     args = map(string.strip, args)
     for i, arg in enumerate(args):
         if len(arg) < 30:
-            raise ValueError("Too short")
+            raise ValueError("too short")
         if not isinstance(arg, unicode):
             arg = unicode(arg)
         args[i] = arg
diff --git a/smartfile/sync.py b/smartfile/sync.py
new file mode 100644
index 0000000..ebf64d2
--- /dev/null
+++ b/smartfile/sync.py
@@ -0,0 +1,171 @@
+import os
+import zlib
+import hashlib
+import tempfile
+
+from itertools import count
+
+
+# Indicates that the block should come from the "server" (source) or "client"
+# (destination) of the sync operation. Any machine can be either client or
+# server, which it is depends on the direction of the sync.
+SRC, DST = 0, 1
+# Default block_size to use.
+BS = 4096
+# Min / Max block_size to use.
+BS_MIN, BS_MAX = 1024, 32768
+# Default amount of file data to buffer in memory before using disk.
+MAX_BUFFER = 1024**2*5
+
+
+class SyncError(Exception):
+    pass
+
+
+def calc_block_size(f):
+    "Tries to obtain the optimal block size."
+    # Try to determine the file size using methods with decreasing chance of
+    # success.
+    size = None
+    # Try to use os.fstat() to determine file size.
+    if callable(getattr(f, 'fileno', None)):
+        try:
+            size = os.fstat(f.fileno()).st_size
+        except:
+            pass
+    # Hmm, try to use seek() to determine file size.
+    if size is None and callable(getattr(f, 'seek', None)):
+        try:
+            f.seek(0, 2)
+            size = f.tell()
+            f.seek(0)
+        except:
+            pass
+    # len()?
+    if size is None:
+        try:
+            size = len(f)
+        except:
+            pass
+    # If we could not determine the size, use the default.
+    if size is None:
+        return BS
+    # Try to use about 1024 blocks, but not less than 1K or greater than 32K.
+    return min(BS_MAX, max(BS_MIN, size / 1024))
+
+
+def checksum(f, block_size=None):
+    """
+    Calculates the rolling checksum of a file. Uses both the fast adler32 and
+    md5 algorithms. Both are used because during delta creation, if the faster
+    adler32 does not match, md5 is skipped. In the case adler32 matches, md5
+    is performed as a stronger "double-check".
+
+    Returns a structure containing file information and the checksums.
+    """
+    if not block_size:
+        block_size = calc_block_size(f)
+    try:
+        f.seek(0)
+    except AttributeError:
+        pass
+    blocks, md5sum = [], hashlib.md5()
+    while True:
+        block = f.read(block_size)
+        md5sum.update(block)
+        if not block:
+            break
+        sum1 = hex(zlib.adler32(block))
+        sum2 = hashlib.md5(block).hexdigest()
+        blocks.append((sum1, sum2))
+    return md5sum.hexdigest(), blocks, block_size
+
+
+def delta(f, md5sum, blocks, block_size=None, max_buffer=MAX_BUFFER):
+    """
+    Uses the rolling checksum of a remote file to generate a delta for the local
+    copy. The result is a structure that instructs how to peice together blocks
+    from the remote file and the local file to create a file that is identical
+    to the local file. Any blocks from the local file that are referenced by
+    this structure will be contained within the blob.
+
+    Returns a two-tuple of the delta structure and a blob containing the
+    referenced blocks.
+
+    For the purposes of this function, our local file is SRC.
+    """
+    if not block_size:
+        block_size = calc_block_size(f)
+    try:
+        f.seek(0)
+    except AttributeError:
+        pass
+    md5sum, blob = hashlib.md5(), tempfile.SpooledTemporaryFile(max_size=max_buffer)
+    i, ranges = 0, []
+    for i in count(0):
+        direction, block = None, f.read(block_size)
+        if block:
+            md5sum.update(block)
+        else:
+            # We ran out of data in our local copy, delta should
+            # instruct copying data from remote file.
+            direction = DST
+        try:
+            sum1, sum2 = blocks[i]
+        except IndexError:
+            if not block:
+                # If we ran out of data AND checksums, we are done.
+                break
+            # We ran out of checksums, delta should instruct copying
+            # data from our local copy.
+            direction = SRC
+        if direction is None:
+            # We have not yet determined direction, meaning, we have a
+            # block and checksums that need to be compared.
+            if (sum1 == hex(zlib.adler32(block)) and
+                sum2 == hashlib.md5(block).hexdigest()):
+                # Data is identical in both copies, delta should instruct
+                # copying data from remote file.
+                direction = DST
+            else:
+                # Data differs, delta should instruct copying data from our
+                # local copy.
+                direction = SRC
+        if direction == DST:
+            offset, length = i*block_size, block_size
+        else:
+            offset, length = blob.tell(), len(block)
+            blob.write(block)
+        ranges.append((direction, offset, length))
+    blob.seek(0)
+    return md5sum.hexdigest(), ranges, blob
+
+
+def patch(f, ranges, blob, max_buffer=MAX_BUFFER):
+    """
+    Applies a delta to the local file by alternately copying data from the
+    local copy and provided blob to recreate the remote file locally.
+
+    After patching it uses the file information to verify that the local file
+    and remote file are identical.
+
+    For the purposes of this function, our local file is DST.
+    """
+    try:
+        f.seek(0)
+    except AttributeError:
+        pass
+    md5sum = hashlib.md5()
+    sources = {
+        SRC: blob,
+        DST: f,
+    }
+    o = tempfile.SpooledTemporaryFile(max_size=max_buffer)
+    for direction, offset, length in ranges:
+        s = sources[direction]
+        s.seek(offset)
+        block = s.read(length)
+        md5sum.update(block)
+        o.write(block)
+    o.seek(0)
+    return o
diff --git a/tests.py b/tests.py
index 1dfa800..7c08100 100644
--- a/tests.py
+++ b/tests.py
@@ -2,20 +2,33 @@
 
 import os
 import json
+import zlib
+import random
+import hashlib
 import urlparse
 import unittest
 import tempfile
 import threading
 
+from StringIO import StringIO
+
 from BaseHTTPServer import HTTPServer
 from BaseHTTPServer import BaseHTTPRequestHandler
 
 from smartfile import BasicClient
 from smartfile import OAuthClient
+from smartfile.sync import checksum
+from smartfile.sync import delta
+from smartfile.sync import patch
+from smartfile.sync import calc_block_size
+from smartfile.sync import SRC
+from smartfile.sync import DST
+from smartfile.sync import BS
+from smartfile.sync import BS_MIN
+from smartfile.sync import BS_MAX
 from smartfile.errors import APIError
 from smartfile.errors import RequestError
 
-
 API_KEY = '8g1aq1UF2QfZTG47yEVhVLAFqyfDdp'
 API_PASSWORD = '3II3UFD3pBAwy3Rbz8mVWBhJTA2Gvd'
 CLIENT_TOKEN = '8oWot4KrppJDzfokDsHNJrND0Ay13s'
@@ -371,5 +384,205 @@ class OAuthJSONTestCase(JSONTestCase, OAuthTestCase):
 # http://stackoverflow.com/questions/2481511/mocking-importerror-in-python
 
 
+class SyncBlockSizeTestCase(unittest.TestCase):
+    def test_fstat(self):
+        "Ensure something that can be fstat()ed is."
+        f = tempfile.NamedTemporaryFile()
+        f.write(os.urandom(2000*1024))
+        f.seek(0)
+        self.assertEqual(calc_block_size(f), 2000)
+
+    def test_seek(self):
+        "Ensure something that can be seek()ed is."
+        f = StringIO()
+        f.write(os.urandom(2001*1024))
+        f.seek(0)
+        self.assertEqual(calc_block_size(f), 2001)
+
+    def test_len(self):
+        "Ensure something that can be len()ed is."
+        class LenableLikeFile(object):
+            "File-like in that it is read()able, but also len()able."
+            def __init__(self, buffer):
+                self.pos = 0
+                self.buffer = buffer
+
+            def __len__(self):
+                return len(self.buffer)
+
+            def read(self, bytes=-1):
+                if bytes == -1:
+                    bytes = len(self.buffer)
+                data = self.buffer[self.pos:self.pos+bytes]
+                self.pos += bytes
+                return data
+
+        f = LenableLikeFile(os.urandom(2002*1024))
+        self.assertEqual(calc_block_size(f), 2002)
+
+    def test_min(self):
+        "Ensure really small files use the minimum block size."
+        f = StringIO()
+        self.assertEqual(calc_block_size(f), BS_MIN)
+
+    def test_max(self):
+        "Ensure really big files use the maximum block size."
+        class BigFakeFile(object):
+            def seek(self, pos, whence=0):
+                pass
+
+            def tell(self):
+                # Mu-ha-ha-ha
+                return BS_MAX*BS_MAX
+
+        f = BigFakeFile()
+        self.assertEqual(calc_block_size(f), BS_MAX)
+
+    def test_default(self):
+        "Ensure files of indeterminate size use the default block size."
+        f = object()
+        self.assertEqual(calc_block_size(f), BS)
+
+
+class SyncChecksumTestCase(unittest.TestCase):
+    def setUp(self):
+        "Create a buffer containing random data."
+        self.rand = StringIO(os.urandom(1024**2))
+
+    def test_block_size_default(self):
+        "Ensure default block_size works."
+        md5sum, blocks, block_size = checksum(self.rand)
+        self.assertEqual(len(blocks), 1024**2/block_size)
+
+    def test_block_size_1024(self):
+        "Ensure the proper number of blocks are produced."
+        md5sum, blocks, block_size = checksum(self.rand, block_size=1024)
+        self.assertEqual(len(blocks), 1024)
+
+    def test_block_size_2048(self):
+        "Ensure the proper number of blocks are produced."
+        md5sum, blocks, block_size = checksum(self.rand, block_size=2048)
+        self.assertEqual(len(blocks), 512)
+
+    def test_blocks(self):
+        "Ensure the block checksums are correct."
+        md5sum, blocks, block_size = checksum(self.rand, block_size=2048)
+        self.rand.seek(0)
+        for i, (sum1, sum2) in enumerate(blocks):
+            block = self.rand.read(2048)
+            self.assertEqual(sum1, hex(zlib.adler32(block)), 'Invalid adler32 sum for block %s' % i)
+            self.assertEqual(sum2, hashlib.md5(block).hexdigest(), 'Invalid md5sum for block %s' % i)
+
+    def test_md5sum(self):
+        "Ensure the file checksum is correct."
+        md5sum, blocks, block_size = checksum(self.rand, block_size=1024)
+        self.assertEqual(md5sum, hashlib.md5(self.rand.getvalue()).hexdigest())
+
+
+class SyncDeltaTestCase(unittest.TestCase):
+    def setUp(self):
+        self.rand1 = StringIO(os.urandom(1024**2))
+        self.rand2 = StringIO(os.urandom(1024**2))
+
+    def test_block_size_default(self):
+        "Ensure default block_size works."
+        md5sum1, blocks, block_size = checksum(self.rand1)
+        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        self.assertEqual(len(ranges), 1024**2/block_size)
+
+    def test_block_size_1024(self):
+        "Ensure block_size of 1024 works."
+        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
+        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        self.assertEqual(len(ranges), 1024)
+
+    def test_block_size_2048(self):
+        "Ensure block_size of 2048 works."
+        md5sum1, blocks, block_size = checksum(self.rand1, block_size=2048)
+        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        self.assertEqual(len(ranges), 512)
+
+    def test_identical(self):
+        "Ensure two files with ALL matching blocks are handled."
+        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
+        md5sum2, ranges, blob = delta(self.rand1, md5sum1, blocks, block_size=block_size)
+        self.assertEqual(md5sum1, md5sum2)
+        self.assertEqual(md5sum2, hashlib.md5(self.rand1.getvalue()).hexdigest())
+        for i, (direction, offset, length) in enumerate(ranges):
+            self.assertEqual(direction, DST, 'Invalid direction %s for block %s' % (direction, i))
+            self.assertEqual(offset, i*1024, 'Invalid offset %s for block %s' % (offset, i))
+            self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
+
+    def test_different(self):
+        "Ensure two files with NO matching blocks are handled."
+        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
+        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        self.assertNotEqual(md5sum1, md5sum2)
+        self.assertEqual(md5sum2, hashlib.md5(self.rand2.getvalue()).hexdigest())
+        for i, (direction, offset, length) in enumerate(ranges):
+            self.assertEqual(direction, SRC, 'Invalid direction %s for block %s' % (direction, i))
+            self.assertEqual(offset, i*1024, 'Invalid offset %s for block %s' % (offset, i))
+            self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
+
+    def test_mixed(self):
+        "Ensure two files with some overlapping blocks are handled."
+        # Make sure first and last blocks match.
+        matching = [0, 1024]
+        # Pick 510 additional random blocks to make identical (half).
+        for i in xrange(510):
+            while True:
+                block_num = random.randint(0, 1024)
+                if block_num not in matching:
+                    break
+            matching.append(block_num)
+        # Copy our matching blocks from SRC to DST
+        for block_num in matching:
+            self.rand1.seek(block_num*1024)
+            self.rand2.seek(block_num*1024)
+            self.rand2.write(self.rand1.read(1024))
+        # Continue as normal.
+        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
+        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        self.assertNotEqual(md5sum1, md5sum2)
+        self.assertEqual(md5sum2, hashlib.md5(self.rand2.getvalue()).hexdigest())
+        for i, (direction, offset, length) in enumerate(ranges):
+            if i in matching:
+                # If the block matches, we will find it in the DST file, which
+                # is the local file. The offset will be the same as the
+                # position it will be written to.
+                d, o = DST, i * 1024
+            else:
+                # If the block differs, we will find it in the blob from the SRC
+                # file. It's offset will be equal to the number of non-matching
+                # blocks so far * block_size.
+                # Count non-matching blocks so far, each will be present in blob.
+                nm = len([b for b in xrange(i) if b not in matching])
+                # Expect SRC as direction and calculate our offset.
+                d, o = SRC, nm * 1024
+            self.assertEqual(direction, d, 'Invalid direction %s for block %s' % (direction, i))
+            self.assertEqual(offset, o, 'Invalid offset %s for block %s' % (offset, i))
+            self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
+
+
+class SyncPatchTestCase(unittest.TestCase):
+    def setUp(self):
+        self.rand1 = StringIO(os.urandom(1024**2))
+        self.rand2 = StringIO(os.urandom(1024**2))
+
+    def test_patch_1024(self):
+        "Ensure a block_size of 1024 works."
+        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
+        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        out = patch(self.rand1, ranges, blob)
+        self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum2)
+
+    def test_patch_2048(self):
+        "Ensure a block_size of 2048 works."
+        md5sum1, blocks, block_size = checksum(self.rand1, block_size=2048)
+        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        out = patch(self.rand1, ranges, blob)
+        self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum2)
+
+
 if __name__ == '__main__':
     unittest.main()

From 542c4516b9a42af5281be4532ded7fdb6690e568 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 30 Apr 2013 23:26:57 -0400
Subject: [PATCH 02/90] Make pep8 happy!

---
 smartfile/sync.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/smartfile/sync.py b/smartfile/sync.py
index ebf64d2..18c2c1b 100644
--- a/smartfile/sync.py
+++ b/smartfile/sync.py
@@ -15,7 +15,7 @@
 # Min / Max block_size to use.
 BS_MIN, BS_MAX = 1024, 32768
 # Default amount of file data to buffer in memory before using disk.
-MAX_BUFFER = 1024**2*5
+MAX_BUFFER = 1024 ** 2 * 5
 
 
 class SyncError(Exception):
@@ -122,8 +122,8 @@ def delta(f, md5sum, blocks, block_size=None, max_buffer=MAX_BUFFER):
         if direction is None:
             # We have not yet determined direction, meaning, we have a
             # block and checksums that need to be compared.
-            if (sum1 == hex(zlib.adler32(block)) and
-                sum2 == hashlib.md5(block).hexdigest()):
+            if sum1 == hex(zlib.adler32(block)) and \
+               sum2 == hashlib.md5(block).hexdigest():
                 # Data is identical in both copies, delta should instruct
                 # copying data from remote file.
                 direction = DST
@@ -132,7 +132,7 @@ def delta(f, md5sum, blocks, block_size=None, max_buffer=MAX_BUFFER):
                 # local copy.
                 direction = SRC
         if direction == DST:
-            offset, length = i*block_size, block_size
+            offset, length = i * block_size, block_size
         else:
             offset, length = blob.tell(), len(block)
             blob.write(block)

From de78c581f0476bb398270e253ac376bbea8da3a5 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Wed, 1 May 2013 14:20:51 -0400
Subject: [PATCH 03/90] Re-implement the sync algorithm to use the rsync
 algorithm (rolling checksum). This version is pretty slow, but very bandwidth
 efficient. Next up is some profiling to speed this up a bit.

---
 smartfile/sync.py | 200 +++++++++++++++++++++++++---------------------
 tests.py          | 157 ++++++++++--------------------------
 2 files changed, 150 insertions(+), 207 deletions(-)

diff --git a/smartfile/sync.py b/smartfile/sync.py
index 18c2c1b..0de1d2d 100644
--- a/smartfile/sync.py
+++ b/smartfile/sync.py
@@ -3,6 +3,7 @@
 import hashlib
 import tempfile
 
+from collections import deque
 from itertools import count
 
 
@@ -12,8 +13,6 @@
 SRC, DST = 0, 1
 # Default block_size to use.
 BS = 4096
-# Min / Max block_size to use.
-BS_MIN, BS_MAX = 1024, 32768
 # Default amount of file data to buffer in memory before using disk.
 MAX_BUFFER = 1024 ** 2 * 5
 
@@ -22,122 +21,140 @@ class SyncError(Exception):
     pass
 
 
-def calc_block_size(f):
-    "Tries to obtain the optimal block size."
-    # Try to determine the file size using methods with decreasing chance of
-    # success.
-    size = None
-    # Try to use os.fstat() to determine file size.
-    if callable(getattr(f, 'fileno', None)):
-        try:
-            size = os.fstat(f.fileno()).st_size
-        except:
-            pass
-    # Hmm, try to use seek() to determine file size.
-    if size is None and callable(getattr(f, 'seek', None)):
-        try:
-            f.seek(0, 2)
-            size = f.tell()
-            f.seek(0)
-        except:
-            pass
-    # len()?
-    if size is None:
-        try:
-            size = len(f)
-        except:
-            pass
-    # If we could not determine the size, use the default.
-    if size is None:
-        return BS
-    # Try to use about 1024 blocks, but not less than 1K or greater than 32K.
-    return min(BS_MAX, max(BS_MIN, size / 1024))
-
-
-def checksum(f, block_size=None):
+class RollingChecksum(object):
+    def __init__(self, data=None, block_size=BS):
+        self.s, self.a, self.b = 0, 0, 0
+        self.block_size = block_size
+        if data:
+            l = len(data)
+            for i in xrange(l):
+                d = ord(data[i])
+                self.a += d
+                self.b += (l - i) * d
+            self.s = (self.b << 16) | self.a
+
+    def roll(self, pop, add):
+        pop, add = ord(pop), ord(add)
+        self.a -= pop - add
+        self.b -= pop * self.block_size - self.a
+        self.s = (self.b << 16) | self.a
+
+    def digest(self):
+        return self.s
+
+
+def table(f, block_size=BS):
     """
-    Calculates the rolling checksum of a file. Uses both the fast adler32 and
-    md5 algorithms. Both are used because during delta creation, if the faster
-    adler32 does not match, md5 is skipped. In the case adler32 matches, md5
-    is performed as a stronger "double-check".
+    Calculates a table containing block checksums for the given stream. These
+    checksums are stored in dictionaries. The first (fast) checksum may have
+    many collisions, so it is probable that multiple blocks will have the same
+    value for the first checksum, but different values for the second.
+
+    {
+        'fast1': {
+            'slow1': (offset, length),
+            'slow2': (offset, length),
+        },
+        'fast2': {
+            'slow3': (offset, length),
+        }
+    }
+
+    So that the faster checksum can yield possible matching blocks. If a match
+    is found at the first level, the slower (MD5) checksum is performed to find
+    the location of the matching block.
 
     Returns a structure containing file information and the checksums.
     """
-    if not block_size:
-        block_size = calc_block_size(f)
     try:
         f.seek(0)
     except AttributeError:
         pass
-    blocks, md5sum = [], hashlib.md5()
+    blocks, md5sum = {}, hashlib.md5()
     while True:
-        block = f.read(block_size)
+        offset, block = f.tell(), f.read(block_size)
         md5sum.update(block)
         if not block:
             break
-        sum1 = hex(zlib.adler32(block))
+        length = len(block)
+        sum1 = RollingChecksum(block).digest()
         sum2 = hashlib.md5(block).hexdigest()
-        blocks.append((sum1, sum2))
-    return md5sum.hexdigest(), blocks, block_size
+        blocks.setdefault(sum1, {})[sum2] = (offset, length)
+    return md5sum.hexdigest(), blocks
 
 
-def delta(f, md5sum, blocks, block_size=None, max_buffer=MAX_BUFFER):
+def delta(f, blocks, block_size=BS, max_buffer=MAX_BUFFER):
     """
-    Uses the rolling checksum of a remote file to generate a delta for the local
-    copy. The result is a structure that instructs how to peice together blocks
-    from the remote file and the local file to create a file that is identical
-    to the local file. Any blocks from the local file that are referenced by
-    this structure will be contained within the blob.
+    Uses the block table of a remote file to generate a delta for the local
+    copy. The stream is scanned one byte at a time while calculating a rolling
+    checksum. At each step, the block list is searched for a match. If a match
+    is found, the slower MD5 sum is used to verify a matching block.
 
     Returns a two-tuple of the delta structure and a blob containing the
     referenced blocks.
 
     For the purposes of this function, our local file is SRC.
     """
-    if not block_size:
-        block_size = calc_block_size(f)
     try:
         f.seek(0)
     except AttributeError:
         pass
-    md5sum, blob = hashlib.md5(), tempfile.SpooledTemporaryFile(max_size=max_buffer)
-    i, ranges = 0, []
-    for i in count(0):
-        direction, block = None, f.read(block_size)
-        if block:
-            md5sum.update(block)
-        else:
-            # We ran out of data in our local copy, delta should
-            # instruct copying data from remote file.
-            direction = DST
-        try:
-            sum1, sum2 = blocks[i]
-        except IndexError:
+    # Ranges will contain a list of ranges to read from SRC or DST to
+    # reassemble the SRC file. Blob contains the referenced ranges from
+    # the SRC, so that the DST can apply them.
+    ranges, blob = [], tempfile.SpooledTemporaryFile(max_size=max_buffer)
+    # Window is the current block we are searching for. Reverse is our write
+    # buffer, data that was examined and fell out of our window.
+    window, reverse = deque(), []
+    # We will be calculating checksums as we move through the stream.
+    sum1, md5sum = RollingChecksum(), hashlib.md5()
+    while True:
+        if not window:
+            block = f.read(block_size)
             if not block:
-                # If we ran out of data AND checksums, we are done.
                 break
-            # We ran out of checksums, delta should instruct copying
-            # data from our local copy.
-            direction = SRC
-        if direction is None:
-            # We have not yet determined direction, meaning, we have a
-            # block and checksums that need to be compared.
-            if sum1 == hex(zlib.adler32(block)) and \
-               sum2 == hashlib.md5(block).hexdigest():
-                # Data is identical in both copies, delta should instruct
-                # copying data from remote file.
-                direction = DST
-            else:
-                # Data differs, delta should instruct copying data from our
-                # local copy.
-                direction = SRC
-        if direction == DST:
-            offset, length = i * block_size, block_size
-        else:
-            offset, length = blob.tell(), len(block)
-            blob.write(block)
-        ranges.append((direction, offset, length))
-    blob.seek(0)
+            window.extend(block)
+            md5sum.update(block)
+            sum1 = RollingChecksum(block)
+        # Check if our window matches any blocks:
+        matches = blocks.get(sum1.digest())
+        if matches:
+            sum2 = hashlib.md5(''.join(window)).hexdigest()
+            match = matches.get(sum2)
+            if match:
+                # We found a block that matches our window.
+                if reverse:
+                    # First flush our reverse buffer.
+                    ranges.append((SRC, blob.tell(), len(reverse)))
+                    blob.write(''.join(reverse))
+                    del reverse[:]
+                elif ranges and ranges[-1][0] == DST:
+                    # If the previous range is also of type DST, merge and
+                    # replace it.
+                    p = ranges.pop()
+                    match = (p[0], p[1] + match[1])
+                ranges.append((DST, ) + match)
+                # dump our window
+                window.clear()
+                continue
+        nbyte = f.read(1)
+        if not nbyte:
+            break
+        md5sum.update(nbyte)
+        # Start moving our window by popping it's tail.
+        obyte = window.popleft()
+        # Update rolling checksum.
+        sum1.roll(obyte, nbyte)
+        # Finish moving our window, appending to head.
+        window.append(nbyte)
+        # The old byte should be written to the blob
+        reverse.append(obyte)
+    # Combine our remaining buffers.
+    reverse.extend(window)
+    # Flush any remaining data.
+    if reverse:
+        ranges.append((SRC, blob.tell(), len(reverse)))
+        blob.write(''.join(reverse))
     return md5sum.hexdigest(), ranges, blob
 
 
@@ -146,9 +163,6 @@ def patch(f, ranges, blob, max_buffer=MAX_BUFFER):
     Applies a delta to the local file by alternately copying data from the
     local copy and provided blob to recreate the remote file locally.
 
-    After patching it uses the file information to verify that the local file
-    and remote file are identical.
-
     For the purposes of this function, our local file is DST.
     """
     try:
diff --git a/tests.py b/tests.py
index 7c08100..72bde22 100644
--- a/tests.py
+++ b/tests.py
@@ -17,15 +17,13 @@
 
 from smartfile import BasicClient
 from smartfile import OAuthClient
-from smartfile.sync import checksum
+from smartfile.sync import table
 from smartfile.sync import delta
 from smartfile.sync import patch
-from smartfile.sync import calc_block_size
+from smartfile.sync import RollingChecksum
 from smartfile.sync import SRC
 from smartfile.sync import DST
 from smartfile.sync import BS
-from smartfile.sync import BS_MIN
-from smartfile.sync import BS_MAX
 from smartfile.errors import APIError
 from smartfile.errors import RequestError
 
@@ -384,98 +382,41 @@ class OAuthJSONTestCase(JSONTestCase, OAuthTestCase):
 # http://stackoverflow.com/questions/2481511/mocking-importerror-in-python
 
 
-class SyncBlockSizeTestCase(unittest.TestCase):
-    def test_fstat(self):
-        "Ensure something that can be fstat()ed is."
-        f = tempfile.NamedTemporaryFile()
-        f.write(os.urandom(2000*1024))
-        f.seek(0)
-        self.assertEqual(calc_block_size(f), 2000)
-
-    def test_seek(self):
-        "Ensure something that can be seek()ed is."
-        f = StringIO()
-        f.write(os.urandom(2001*1024))
-        f.seek(0)
-        self.assertEqual(calc_block_size(f), 2001)
-
-    def test_len(self):
-        "Ensure something that can be len()ed is."
-        class LenableLikeFile(object):
-            "File-like in that it is read()able, but also len()able."
-            def __init__(self, buffer):
-                self.pos = 0
-                self.buffer = buffer
-
-            def __len__(self):
-                return len(self.buffer)
-
-            def read(self, bytes=-1):
-                if bytes == -1:
-                    bytes = len(self.buffer)
-                data = self.buffer[self.pos:self.pos+bytes]
-                self.pos += bytes
-                return data
-
-        f = LenableLikeFile(os.urandom(2002*1024))
-        self.assertEqual(calc_block_size(f), 2002)
-
-    def test_min(self):
-        "Ensure really small files use the minimum block size."
-        f = StringIO()
-        self.assertEqual(calc_block_size(f), BS_MIN)
-
-    def test_max(self):
-        "Ensure really big files use the maximum block size."
-        class BigFakeFile(object):
-            def seek(self, pos, whence=0):
-                pass
-
-            def tell(self):
-                # Mu-ha-ha-ha
-                return BS_MAX*BS_MAX
-
-        f = BigFakeFile()
-        self.assertEqual(calc_block_size(f), BS_MAX)
-
-    def test_default(self):
-        "Ensure files of indeterminate size use the default block size."
-        f = object()
-        self.assertEqual(calc_block_size(f), BS)
-
-
-class SyncChecksumTestCase(unittest.TestCase):
+class SyncTableTestCase(unittest.TestCase):
     def setUp(self):
         "Create a buffer containing random data."
         self.rand = StringIO(os.urandom(1024**2))
 
     def test_block_size_default(self):
         "Ensure default block_size works."
-        md5sum, blocks, block_size = checksum(self.rand)
-        self.assertEqual(len(blocks), 1024**2/block_size)
+        md5sum, blocks = table(self.rand)
+        self.assertEqual(len(blocks), self.rand.tell()/BS)
 
     def test_block_size_1024(self):
         "Ensure the proper number of blocks are produced."
-        md5sum, blocks, block_size = checksum(self.rand, block_size=1024)
+        md5sum, blocks = table(self.rand, block_size=1024)
         self.assertEqual(len(blocks), 1024)
 
     def test_block_size_2048(self):
         "Ensure the proper number of blocks are produced."
-        md5sum, blocks, block_size = checksum(self.rand, block_size=2048)
+        md5sum, blocks = table(self.rand, block_size=2048)
         self.assertEqual(len(blocks), 512)
 
     def test_blocks(self):
         "Ensure the block checksums are correct."
-        md5sum, blocks, block_size = checksum(self.rand, block_size=2048)
+        md5sum, blocks = table(self.rand)
         self.rand.seek(0)
-        for i, (sum1, sum2) in enumerate(blocks):
-            block = self.rand.read(2048)
-            self.assertEqual(sum1, hex(zlib.adler32(block)), 'Invalid adler32 sum for block %s' % i)
-            self.assertEqual(sum2, hashlib.md5(block).hexdigest(), 'Invalid md5sum for block %s' % i)
+        while True:
+            block = self.rand.read(BS)
+            if not block:
+                break
+            match = blocks.get(RollingChecksum(block).digest())
+            self.assertIsNotNone(match)
+            self.assertIn(hashlib.md5(block).hexdigest(), match)
 
     def test_md5sum(self):
         "Ensure the file checksum is correct."
-        md5sum, blocks, block_size = checksum(self.rand, block_size=1024)
+        md5sum, blocks = table(self.rand, block_size=1024)
         self.assertEqual(md5sum, hashlib.md5(self.rand.getvalue()).hexdigest())
 
 
@@ -484,45 +425,27 @@ def setUp(self):
         self.rand1 = StringIO(os.urandom(1024**2))
         self.rand2 = StringIO(os.urandom(1024**2))
 
-    def test_block_size_default(self):
-        "Ensure default block_size works."
-        md5sum1, blocks, block_size = checksum(self.rand1)
-        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
-        self.assertEqual(len(ranges), 1024**2/block_size)
-
-    def test_block_size_1024(self):
-        "Ensure block_size of 1024 works."
-        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
-        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
-        self.assertEqual(len(ranges), 1024)
-
-    def test_block_size_2048(self):
-        "Ensure block_size of 2048 works."
-        md5sum1, blocks, block_size = checksum(self.rand1, block_size=2048)
-        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
-        self.assertEqual(len(ranges), 512)
-
     def test_identical(self):
         "Ensure two files with ALL matching blocks are handled."
-        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
-        md5sum2, ranges, blob = delta(self.rand1, md5sum1, blocks, block_size=block_size)
+        md5sum1, blocks = table(self.rand1)
+        md5sum2, ranges, blob = delta(self.rand1, blocks)
         self.assertEqual(md5sum1, md5sum2)
         self.assertEqual(md5sum2, hashlib.md5(self.rand1.getvalue()).hexdigest())
-        for i, (direction, offset, length) in enumerate(ranges):
-            self.assertEqual(direction, DST, 'Invalid direction %s for block %s' % (direction, i))
-            self.assertEqual(offset, i*1024, 'Invalid offset %s for block %s' % (offset, i))
-            self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
+        # No non-matching blocks
+        self.assertEqual(blob.tell(), 0)
+        # All blocks should be collapsed into a single range
+        self.assertEqual(len(ranges), 1)
 
     def test_different(self):
         "Ensure two files with NO matching blocks are handled."
-        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
-        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        md5sum1, blocks = table(self.rand1)
+        md5sum2, ranges, blob = delta(self.rand2, blocks)
         self.assertNotEqual(md5sum1, md5sum2)
         self.assertEqual(md5sum2, hashlib.md5(self.rand2.getvalue()).hexdigest())
-        for i, (direction, offset, length) in enumerate(ranges):
-            self.assertEqual(direction, SRC, 'Invalid direction %s for block %s' % (direction, i))
-            self.assertEqual(offset, i*1024, 'Invalid offset %s for block %s' % (offset, i))
-            self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
+        # Blob should contain the entire source file
+        self.assertEqual(blob.tell(), self.rand1.tell())
+        # There should be one range representing the entire source file
+        self.assertEqual(len(ranges), 1)
 
     def test_mixed(self):
         "Ensure two files with some overlapping blocks are handled."
@@ -541,8 +464,8 @@ def test_mixed(self):
             self.rand2.seek(block_num*1024)
             self.rand2.write(self.rand1.read(1024))
         # Continue as normal.
-        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
-        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        md5sum1, blocks = table(self.rand1, block_size=1024)
+        md5sum2, ranges, blob = delta(self.rand2, blocks, block_size=1024)
         self.assertNotEqual(md5sum1, md5sum2)
         self.assertEqual(md5sum2, hashlib.md5(self.rand2.getvalue()).hexdigest())
         for i, (direction, offset, length) in enumerate(ranges):
@@ -561,7 +484,7 @@ def test_mixed(self):
                 d, o = SRC, nm * 1024
             self.assertEqual(direction, d, 'Invalid direction %s for block %s' % (direction, i))
             self.assertEqual(offset, o, 'Invalid offset %s for block %s' % (offset, i))
-            self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
+            #self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
 
 
 class SyncPatchTestCase(unittest.TestCase):
@@ -569,20 +492,26 @@ def setUp(self):
         self.rand1 = StringIO(os.urandom(1024**2))
         self.rand2 = StringIO(os.urandom(1024**2))
 
-    def test_patch_1024(self):
+    def test_1024(self):
         "Ensure a block_size of 1024 works."
-        md5sum1, blocks, block_size = checksum(self.rand1, block_size=1024)
-        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        md5sum1, blocks = table(self.rand1, block_size=1024)
+        md5sum2, ranges, blob = delta(self.rand2, blocks, block_size=1024)
         out = patch(self.rand1, ranges, blob)
         self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum2)
 
-    def test_patch_2048(self):
+    def test_2048(self):
         "Ensure a block_size of 2048 works."
-        md5sum1, blocks, block_size = checksum(self.rand1, block_size=2048)
-        md5sum2, ranges, blob = delta(self.rand2, md5sum1, blocks, block_size=block_size)
+        md5sum1, blocks = table(self.rand1, block_size=2048)
+        md5sum2, ranges, blob = delta(self.rand2, blocks, block_size=1024)
         out = patch(self.rand1, ranges, blob)
         self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum2)
 
+    def test_identical(self):
+        md5sum1, blocks = table(self.rand1, block_size=2048)
+        md5sum2, ranges, blob = delta(self.rand1, blocks, block_size=1024)
+        out = patch(self.rand1, ranges, blob)
+        self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum1)
+
 
 if __name__ == '__main__':
     unittest.main()

From 8cce7329049c92c4685deca7a66de7c549cab2d7 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Wed, 1 May 2013 17:09:34 -0400
Subject: [PATCH 04/90] Add ability to easily profile the sync functions.

---
 Makefile   |  4 ++++
 profile.py | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 profile.py

diff --git a/Makefile b/Makefile
index ec392ec..0488a3f 100644
--- a/Makefile
+++ b/Makefile
@@ -11,3 +11,7 @@ install:
 publish:
 	python setup.py register
 	python setup.py sdist upload
+
+profile:
+	python profile.py
+
diff --git a/profile.py b/profile.py
new file mode 100644
index 0000000..7171309
--- /dev/null
+++ b/profile.py
@@ -0,0 +1,19 @@
+import os
+import cProfile
+
+from StringIO import StringIO
+
+from smartfile import sync
+
+
+s1 = StringIO(os.urandom(1024**2))
+s2 = StringIO(os.urandom(1024**2))
+
+#blocks = sync.table(s1)
+cProfile.run('blocks = sync.table(s1)')
+
+#ranges, blob = sync.delta(s2, blocks)
+cProfile.run('ranges, blob = sync.delta(s2, blocks)')
+
+#out = sync.patch(s1, ranges, blob)
+cProfile.run('out = sync.patch(s1, ranges, blob)')
\ No newline at end of file

From df53f26f40cbebf4bff59d83a6326614b15ae211 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Wed, 1 May 2013 17:09:51 -0400
Subject: [PATCH 05/90] Fixed some small bugs. Profiling yielded a 50%
 improvement for table() and 30% for delta(). These functions are still not
 FAST, but much better.

---
 smartfile/sync.py |  67 +++++++++++-----------
 tests.py          | 139 +++++++++++++++++++++++++++++++---------------
 2 files changed, 124 insertions(+), 82 deletions(-)

diff --git a/smartfile/sync.py b/smartfile/sync.py
index 0de1d2d..2aa683a 100644
--- a/smartfile/sync.py
+++ b/smartfile/sync.py
@@ -23,24 +23,20 @@ class SyncError(Exception):
 
 class RollingChecksum(object):
     def __init__(self, data=None, block_size=BS):
-        self.s, self.a, self.b = 0, 0, 0
-        self.block_size = block_size
+        self.bs, self.s, self.a, self.b = block_size, 0, 0, 0
         if data:
-            l = len(data)
-            for i in xrange(l):
-                d = ord(data[i])
-                self.a += d
+            l, data = len(data), map(ord, data)
+            self.a = sum(data)
+            for i, d in enumerate(data):
                 self.b += (l - i) * d
-            self.s = (self.b << 16) | self.a
 
     def roll(self, pop, add):
         pop, add = ord(pop), ord(add)
         self.a -= pop - add
-        self.b -= pop * self.block_size - self.a
-        self.s = (self.b << 16) | self.a
+        self.b -= pop * self.bs - self.a
 
     def digest(self):
-        return self.s
+        return (self.b << 16) | self.a
 
 
 def table(f, block_size=BS):
@@ -70,17 +66,16 @@ def table(f, block_size=BS):
         f.seek(0)
     except AttributeError:
         pass
-    blocks, md5sum = {}, hashlib.md5()
+    blocks = {}
     while True:
         offset, block = f.tell(), f.read(block_size)
-        md5sum.update(block)
         if not block:
             break
         length = len(block)
         sum1 = RollingChecksum(block).digest()
         sum2 = hashlib.md5(block).hexdigest()
         blocks.setdefault(sum1, {})[sum2] = (offset, length)
-    return md5sum.hexdigest(), blocks
+    return blocks
 
 
 def delta(f, blocks, block_size=BS, max_buffer=MAX_BUFFER):
@@ -104,24 +99,26 @@ def delta(f, blocks, block_size=BS, max_buffer=MAX_BUFFER):
     # the SRC, so that the DST can apply them.
     ranges, blob = [], tempfile.SpooledTemporaryFile(max_size=max_buffer)
     # Window is the current block we are searching for. Reverse is our write
-    # buffer, data that was examined and fell out of our window.
-    window, reverse = deque(), []
+    # buffer, data that was examined and fell out of our window. Forward is our
+    # read buffer, allowing us to read more than one byte at a time.
+    window, forward, reverse = deque(), deque(), []
     # We will be calculating checksums as we move through the stream.
-    sum1, md5sum = RollingChecksum(), hashlib.md5()
+    sum1 = RollingChecksum()
     while True:
         if not window:
-            block = f.read(block_size)
-            if not block:
-                break
-            window.extend(block)
-            md5sum.update(block)
-            sum1 = RollingChecksum(block)
+            if forward:
+                window.extend(forward)
+                forward.clear()
+            need = block_size - len(window)
+            if need:
+                window.extend(f.read(need))
+            sum1 = RollingChecksum(''.join(window), block_size=block_size)
         # Check if our window matches any blocks:
         matches = blocks.get(sum1.digest())
         if matches:
             sum2 = hashlib.md5(''.join(window)).hexdigest()
-            match = matches.get(sum2)
-            if match:
+            offset, length = matches.get(sum2, (None, None))
+            if offset is not None:
                 # We found a block that matches our window.
                 if reverse:
                     # First flush our reverse buffer.
@@ -131,16 +128,17 @@ def delta(f, blocks, block_size=BS, max_buffer=MAX_BUFFER):
                 elif ranges and ranges[-1][0] == DST:
                     # If the previous range is also of type DST, merge and
                     # replace it.
-                    p = ranges.pop()
-                    match = (p[0], p[1] + match[1])
-                ranges.append((DST, ) + match)
+                    _, poffset, plength = ranges.pop()
+                    offset, length = poffset, plength + length
+                ranges.append((DST, offset, length))
                 # dump our window
                 window.clear()
                 continue
-        nbyte = f.read(1)
-        if not nbyte:
-            break
-        md5sum.update(nbyte)
+        if not forward:
+            forward.extend(f.read(block_size))
+            if not forward:
+                break
+        nbyte = forward.popleft()
         # Start moving our window by popping it's tail.
         obyte = window.popleft()
         # Update rolling checksum.
@@ -155,7 +153,7 @@ def delta(f, blocks, block_size=BS, max_buffer=MAX_BUFFER):
     if reverse:
         ranges.append((SRC, blob.tell(), len(reverse)))
         blob.write(''.join(reverse))
-    return md5sum.hexdigest(), ranges, blob
+    return ranges, blob
 
 
 def patch(f, ranges, blob, max_buffer=MAX_BUFFER):
@@ -169,7 +167,6 @@ def patch(f, ranges, blob, max_buffer=MAX_BUFFER):
         f.seek(0)
     except AttributeError:
         pass
-    md5sum = hashlib.md5()
     sources = {
         SRC: blob,
         DST: f,
@@ -178,8 +175,6 @@ def patch(f, ranges, blob, max_buffer=MAX_BUFFER):
     for direction, offset, length in ranges:
         s = sources[direction]
         s.seek(offset)
-        block = s.read(length)
-        md5sum.update(block)
-        o.write(block)
+        o.write(s.read(length))
     o.seek(0)
     return o
diff --git a/tests.py b/tests.py
index 72bde22..2734139 100644
--- a/tests.py
+++ b/tests.py
@@ -10,6 +10,8 @@
 import tempfile
 import threading
 
+from collections import deque
+
 from StringIO import StringIO
 
 from BaseHTTPServer import HTTPServer
@@ -389,22 +391,22 @@ def setUp(self):
 
     def test_block_size_default(self):
         "Ensure default block_size works."
-        md5sum, blocks = table(self.rand)
+        blocks = table(self.rand)
         self.assertEqual(len(blocks), self.rand.tell()/BS)
 
     def test_block_size_1024(self):
         "Ensure the proper number of blocks are produced."
-        md5sum, blocks = table(self.rand, block_size=1024)
+        blocks = table(self.rand, block_size=1024)
         self.assertEqual(len(blocks), 1024)
 
     def test_block_size_2048(self):
         "Ensure the proper number of blocks are produced."
-        md5sum, blocks = table(self.rand, block_size=2048)
+        blocks = table(self.rand, block_size=2048)
         self.assertEqual(len(blocks), 512)
 
     def test_blocks(self):
         "Ensure the block checksums are correct."
-        md5sum, blocks = table(self.rand)
+        blocks = table(self.rand)
         self.rand.seek(0)
         while True:
             block = self.rand.read(BS)
@@ -414,11 +416,6 @@ def test_blocks(self):
             self.assertIsNotNone(match)
             self.assertIn(hashlib.md5(block).hexdigest(), match)
 
-    def test_md5sum(self):
-        "Ensure the file checksum is correct."
-        md5sum, blocks = table(self.rand, block_size=1024)
-        self.assertEqual(md5sum, hashlib.md5(self.rand.getvalue()).hexdigest())
-
 
 class SyncDeltaTestCase(unittest.TestCase):
     def setUp(self):
@@ -427,10 +424,8 @@ def setUp(self):
 
     def test_identical(self):
         "Ensure two files with ALL matching blocks are handled."
-        md5sum1, blocks = table(self.rand1)
-        md5sum2, ranges, blob = delta(self.rand1, blocks)
-        self.assertEqual(md5sum1, md5sum2)
-        self.assertEqual(md5sum2, hashlib.md5(self.rand1.getvalue()).hexdigest())
+        blocks = table(self.rand1)
+        ranges, blob = delta(self.rand1, blocks)
         # No non-matching blocks
         self.assertEqual(blob.tell(), 0)
         # All blocks should be collapsed into a single range
@@ -438,10 +433,8 @@ def test_identical(self):
 
     def test_different(self):
         "Ensure two files with NO matching blocks are handled."
-        md5sum1, blocks = table(self.rand1)
-        md5sum2, ranges, blob = delta(self.rand2, blocks)
-        self.assertNotEqual(md5sum1, md5sum2)
-        self.assertEqual(md5sum2, hashlib.md5(self.rand2.getvalue()).hexdigest())
+        blocks = table(self.rand1)
+        ranges, blob = delta(self.rand2, blocks)
         # Blob should contain the entire source file
         self.assertEqual(blob.tell(), self.rand1.tell())
         # There should be one range representing the entire source file
@@ -464,27 +457,17 @@ def test_mixed(self):
             self.rand2.seek(block_num*1024)
             self.rand2.write(self.rand1.read(1024))
         # Continue as normal.
-        md5sum1, blocks = table(self.rand1, block_size=1024)
-        md5sum2, ranges, blob = delta(self.rand2, blocks, block_size=1024)
-        self.assertNotEqual(md5sum1, md5sum2)
-        self.assertEqual(md5sum2, hashlib.md5(self.rand2.getvalue()).hexdigest())
+        blocks = table(self.rand1, block_size=1024)
+        ranges, blob = delta(self.rand2, blocks, block_size=1024)
+        self.assertLess(blob.tell(), self.rand1.tell() / 2 + 4096)
         for i, (direction, offset, length) in enumerate(ranges):
-            if i in matching:
-                # If the block matches, we will find it in the DST file, which
-                # is the local file. The offset will be the same as the
-                # position it will be written to.
-                d, o = DST, i * 1024
-            else:
-                # If the block differs, we will find it in the blob from the SRC
-                # file. It's offset will be equal to the number of non-matching
-                # blocks so far * block_size.
-                # Count non-matching blocks so far, each will be present in blob.
-                nm = len([b for b in xrange(i) if b not in matching])
-                # Expect SRC as direction and calculate our offset.
-                d, o = SRC, nm * 1024
-            self.assertEqual(direction, d, 'Invalid direction %s for block %s' % (direction, i))
-            self.assertEqual(offset, o, 'Invalid offset %s for block %s' % (offset, i))
-            #self.assertEqual(length, 1024, 'Invalid length %s for block %s' % (length, i))
+            if direction == DST:
+                # If the block matches, we should find it's offset / block_size
+                # in matching:
+                block_num = offset / 1024
+                self.assertIn(block_num, matching)
+            # Since blocks are combined, there are not many other assertions we
+            # can make.
 
 
 class SyncPatchTestCase(unittest.TestCase):
@@ -494,23 +477,87 @@ def setUp(self):
 
     def test_1024(self):
         "Ensure a block_size of 1024 works."
-        md5sum1, blocks = table(self.rand1, block_size=1024)
-        md5sum2, ranges, blob = delta(self.rand2, blocks, block_size=1024)
+        blocks = table(self.rand1, block_size=1024)
+        ranges, blob = delta(self.rand2, blocks, block_size=1024)
         out = patch(self.rand1, ranges, blob)
-        self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum2)
+        self.assertEqual(hashlib.md5(out.read()).digest(),
+                         hashlib.md5(self.rand2.getvalue()).digest())
 
     def test_2048(self):
         "Ensure a block_size of 2048 works."
-        md5sum1, blocks = table(self.rand1, block_size=2048)
-        md5sum2, ranges, blob = delta(self.rand2, blocks, block_size=1024)
+        blocks = table(self.rand1, block_size=2048)
+        ranges, blob = delta(self.rand2, blocks, block_size=2048)
+        out = patch(self.rand1, ranges, blob)
+        self.assertEqual(hashlib.md5(out.read()).digest(),
+                         hashlib.md5(self.rand2.getvalue()).digest())
+
+    def test_2001(self):
+        "Ensure an odd block size works."
+        blocks = table(self.rand1, block_size=2001)
+        ranges, blob = delta(self.rand2, blocks, block_size=2001)
         out = patch(self.rand1, ranges, blob)
-        self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum2)
+        self.assertEqual(hashlib.md5(out.read()).digest(),
+                         hashlib.md5(self.rand2.getvalue()).digest())
 
     def test_identical(self):
-        md5sum1, blocks = table(self.rand1, block_size=2048)
-        md5sum2, ranges, blob = delta(self.rand1, blocks, block_size=1024)
+        "Ensure patching a file that is identical works."
+        blocks = table(self.rand1, block_size=2048)
+        ranges, blob = delta(self.rand1, blocks, block_size=1024)
         out = patch(self.rand1, ranges, blob)
-        self.assertEqual(hashlib.md5(out.read()).hexdigest(), md5sum1)
+        self.assertEqual(hashlib.md5(out.read()).digest(),
+                         hashlib.md5(self.rand1.getvalue()).digest())
+
+
+# Original functions, I converted to a class and optimized.
+# http://code.activestate.com/recipes/577518-rsync-algorithm/
+def weakchecksum(data):
+    a = b = 0
+    l = len(data)
+    for i in range(l):
+        a += ord(data[i])
+        b += (l - i)*ord(data[i])
+    return (b << 16) | a, a, b
+
+
+def rollingchecksum(removed, new, a, b, blocksize=4096):
+    a -= ord(removed) - ord(new)
+    b -= ord(removed) * blocksize - a
+    return (b << 16) | a, a, b
+
+
+class SyncChecksumTestCase(unittest.TestCase):
+    def test_single(self):
+        "Ensure our checksum on a static buffer is true to the original."
+        for i in xrange(100):
+            data = os.urandom(32)
+            self.assertEqual(RollingChecksum(data).digest(), weakchecksum(data)[0])
+
+    def test_rolling(self):
+        "Ensure our rolling checksum is true to the original."
+        data = deque(os.urandom(32))
+        sum1 = RollingChecksum(data, block_size=32)
+        sum2, a, b = weakchecksum(data)
+        for i in xrange(100):
+            next = os.urandom(1)
+            data.append(next)
+            last = data.popleft()
+            sum1.roll(last, next)
+            sum2, a, b = rollingchecksum(last, next, a, b, blocksize=32)
+            self.assertEqual(sum1.digest(), sum2)
+
+    def test_equality(self):
+        """Ensure that a rolling checksum created on a buffer is equal to the
+        one created when rolling over that buffer."""
+        data = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+        # Create a checksum of bytes 10 -20
+        sum1 = RollingChecksum(data[10:20], block_size=10)
+        # Create a checksum of bytes 0 - 10
+        sum2 = RollingChecksum(data[:10], block_size=10)
+        # Roll the second checksum over the buffer until reaching bytes 10-20.
+        for i in xrange(10):
+            sum2.roll(data[i], data[10+i])
+        # Sums should now be equal.
+        self.assertEqual(sum1.digest(), sum2.digest())
 
 
 if __name__ == '__main__':

From e384e4adbc8b654efac26e12810a3c982ecab19f Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Sat, 4 May 2013 09:08:08 -0400
Subject: [PATCH 06/90] Use librsync instead of python rsync implementation.

---
 requirements.txt  |   1 +
 setup.py          |   2 +-
 smartfile/sync.py | 235 ++++++++++++++--------------------------------
 tests.py          | 183 ------------------------------------
 4 files changed, 73 insertions(+), 348 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 3201f57..127c969 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
 oauthlib
 requests
 requests_oauthlib
+python-librsync
diff --git a/setup.py b/setup.py
index b2f044c..cfca497 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@ def get_path(path):
 
 
 name = 'smartfile'
-release = '1'
+release = '2'
 versrel = VERSION + '-' + release
 long_description = file(get_path('README.rst')).read()
 
diff --git a/smartfile/sync.py b/smartfile/sync.py
index 2aa683a..a7b60e0 100644
--- a/smartfile/sync.py
+++ b/smartfile/sync.py
@@ -1,180 +1,87 @@
 import os
-import zlib
-import hashlib
-import tempfile
 
-from collections import deque
-from itertools import count
+try:
+    import librsync
+except ImportError:
+    raise ImportError('python-librsync is required for sync capabilities. '
+                      'Install it using `pip install python-librsync`.')
 
 
-# Indicates that the block should come from the "server" (source) or "client"
-# (destination) of the sync operation. Any machine can be either client or
-# server, which it is depends on the direction of the sync.
-SRC, DST = 0, 1
-# Default block_size to use.
-BS = 4096
-# Default amount of file data to buffer in memory before using disk.
-MAX_BUFFER = 1024 ** 2 * 5
-
-
-class SyncError(Exception):
-    pass
+class BaseFile(object):
+    """
+    Base class for files being synchronized.
+    """
+    def __init__(self, path):
+        self.path = path
 
 
-class RollingChecksum(object):
-    def __init__(self, data=None, block_size=BS):
-        self.bs, self.s, self.a, self.b = block_size, 0, 0, 0
-        if data:
-            l, data = len(data), map(ord, data)
-            self.a = sum(data)
-            for i, d in enumerate(data):
-                self.b += (l - i) * d
+class LocalFile(BaseFile):
+    """
+    Represents a local file that is being synchronized. Uses librsync to
+    perform the steps of the rsync algorithm.
+    """
+    def signature(self, block_size=None):
+        kwargs = {}
+        if block_size:
+            kwargs['block_size'] = block_size
+        return librsync.signature(file(self.path, 'rb'), **kwargs)
 
-    def roll(self, pop, add):
-        pop, add = ord(pop), ord(add)
-        self.a -= pop - add
-        self.b -= pop * self.bs - self.a
+    def delta(self, signature):
+        return librsync.delta(file(self.path, 'rb'), signature)
 
-    def digest(self):
-        return (self.b << 16) | self.a
+    def patch(self, delta):
+        # Open the local file, data may be read from it.
+        f = file(self.path, 'rb')
+        # Unlink the local file, it will remain readable.
+        os.unlink(self.path)
+        # Now patch the file to the local path, replacing it.
+        return librsync.patch(f, delta, o=file(self.path, 'wb'))
 
 
-def table(f, block_size=BS):
+class RemoteFile(BaseFile):
     """
-    Calculates a table containing block checksums for the given stream. These
-    checksums are stored in dictionaries. The first (fast) checksum may have
-    many collisions, so it is probable that multiple blocks will have the same
-    value for the first checksum, but different values for the second.
-
-    {
-        'fast1': {
-            'slow1': (offset, length),
-            'slow2': (offset, length),
-        },
-        'fast2': {
-            'slow3': (offset, length),
-        }
-    }
-
-    So that the faster checksum can yield possible matching blocks. If a match
-    is found at the first level, the slower (MD5) checksum is performed to find
-    the location of the matching block.
-
-    Returns a structure containing file information and the checksums.
+    Represents a remote file that is being synchronized. Makes API calls to
+    perform the steps of the rsync algorithm.
     """
-    try:
-        f.seek(0)
-    except AttributeError:
-        pass
-    blocks = {}
-    while True:
-        offset, block = f.tell(), f.read(block_size)
-        if not block:
-            break
-        length = len(block)
-        sum1 = RollingChecksum(block).digest()
-        sum2 = hashlib.md5(block).hexdigest()
-        blocks.setdefault(sum1, {})[sum2] = (offset, length)
-    return blocks
-
-
-def delta(f, blocks, block_size=BS, max_buffer=MAX_BUFFER):
-    """
-    Uses the block table of a remote file to generate a delta for the local
-    copy. The stream is scanned one byte at a time while calculating a rolling
-    checksum. At each step, the block list is searched for a match. If a match
-    is found, the slower MD5 sum is used to verify a matching block.
+    def __init__(self, path, api):
+        super(RemoteFile, self).__init__(path)
+        self.api = api
 
-    Returns a two-tuple of the delta structure and a blob containing the
-    referenced blocks.
+    def signature(self, block_size=None):
+        kwargs = {}
+        if block_size:
+            kwargs['block_size'] = block_size
+        return self.api.get('sync/signature', self.path, **kwargs)
 
-    For the purposes of this function, our local file is SRC.
-    """
-    try:
-        f.seek(0)
-    except AttributeError:
-        pass
-    # Ranges will contain a list of ranges to read from SRC or DST to
-    # reassemble the SRC file. Blob contains the referenced ranges from
-    # the SRC, so that the DST can apply them.
-    ranges, blob = [], tempfile.SpooledTemporaryFile(max_size=max_buffer)
-    # Window is the current block we are searching for. Reverse is our write
-    # buffer, data that was examined and fell out of our window. Forward is our
-    # read buffer, allowing us to read more than one byte at a time.
-    window, forward, reverse = deque(), deque(), []
-    # We will be calculating checksums as we move through the stream.
-    sum1 = RollingChecksum()
-    while True:
-        if not window:
-            if forward:
-                window.extend(forward)
-                forward.clear()
-            need = block_size - len(window)
-            if need:
-                window.extend(f.read(need))
-            sum1 = RollingChecksum(''.join(window), block_size=block_size)
-        # Check if our window matches any blocks:
-        matches = blocks.get(sum1.digest())
-        if matches:
-            sum2 = hashlib.md5(''.join(window)).hexdigest()
-            offset, length = matches.get(sum2, (None, None))
-            if offset is not None:
-                # We found a block that matches our window.
-                if reverse:
-                    # First flush our reverse buffer.
-                    ranges.append((SRC, blob.tell(), len(reverse)))
-                    blob.write(''.join(reverse))
-                    del reverse[:]
-                elif ranges and ranges[-1][0] == DST:
-                    # If the previous range is also of type DST, merge and
-                    # replace it.
-                    _, poffset, plength = ranges.pop()
-                    offset, length = poffset, plength + length
-                ranges.append((DST, offset, length))
-                # dump our window
-                window.clear()
-                continue
-        if not forward:
-            forward.extend(f.read(block_size))
-            if not forward:
-                break
-        nbyte = forward.popleft()
-        # Start moving our window by popping it's tail.
-        obyte = window.popleft()
-        # Update rolling checksum.
-        sum1.roll(obyte, nbyte)
-        # Finish moving our window, appending to head.
-        window.append(nbyte)
-        # The old byte should be written to the blob
-        reverse.append(obyte)
-    # Combine our remaining buffers.
-    reverse.extend(window)
-    # Flush any remaining data.
-    if reverse:
-        ranges.append((SRC, blob.tell(), len(reverse)))
-        blob.write(''.join(reverse))
-    return ranges, blob
-
-
-def patch(f, ranges, blob, max_buffer=MAX_BUFFER):
-    """
-    Applies a delta to the local file by alternately copying data from the
-    local copy and provided blob to recreate the remote file locally.
+    def delta(self, signature):
+        return self.api.post('sync/delta', self.path, signature=signature)
 
-    For the purposes of this function, our local file is DST.
+    def patch(self, delta):
+        return self.api.post('sync/patch', self.path, delta=delta)
+
+
+class SyncClient(object):
+    """
+    Synchronizes remote and local files.
     """
-    try:
-        f.seek(0)
-    except AttributeError:
-        pass
-    sources = {
-        SRC: blob,
-        DST: f,
-    }
-    o = tempfile.SpooledTemporaryFile(max_size=max_buffer)
-    for direction, offset, length in ranges:
-        s = sources[direction]
-        s.seek(offset)
-        o.write(s.read(length))
-    o.seek(0)
-    return o
+    def __init__(self, api, block_size=None):
+        self.api = api
+        self.block_size = block_size
+
+    def sync(self, src, dst):
+        """
+        Performs synchronization from source to destination.
+        """
+        return dst.patch(src.delta(dst.signature(block_size=self.block_size)))
+
+    def sync_to_server(self, local, remote):
+        """
+        Performs synchronization from a local file to a remote file.
+        """
+        self.sync(LocalFile(local), RemoteFile(remote, self.api))
+
+    def sync_from_server(self, local, remote):
+        """
+        Performs synchronization from a remote file to the local system.
+        """
+        new = self.sync(RemoteFile(remote, self.api), LocalFile(local))
diff --git a/tests.py b/tests.py
index 2734139..3a9c2c7 100644
--- a/tests.py
+++ b/tests.py
@@ -19,13 +19,6 @@
 
 from smartfile import BasicClient
 from smartfile import OAuthClient
-from smartfile.sync import table
-from smartfile.sync import delta
-from smartfile.sync import patch
-from smartfile.sync import RollingChecksum
-from smartfile.sync import SRC
-from smartfile.sync import DST
-from smartfile.sync import BS
 from smartfile.errors import APIError
 from smartfile.errors import RequestError
 
@@ -384,181 +377,5 @@ class OAuthJSONTestCase(JSONTestCase, OAuthTestCase):
 # http://stackoverflow.com/questions/2481511/mocking-importerror-in-python
 
 
-class SyncTableTestCase(unittest.TestCase):
-    def setUp(self):
-        "Create a buffer containing random data."
-        self.rand = StringIO(os.urandom(1024**2))
-
-    def test_block_size_default(self):
-        "Ensure default block_size works."
-        blocks = table(self.rand)
-        self.assertEqual(len(blocks), self.rand.tell()/BS)
-
-    def test_block_size_1024(self):
-        "Ensure the proper number of blocks are produced."
-        blocks = table(self.rand, block_size=1024)
-        self.assertEqual(len(blocks), 1024)
-
-    def test_block_size_2048(self):
-        "Ensure the proper number of blocks are produced."
-        blocks = table(self.rand, block_size=2048)
-        self.assertEqual(len(blocks), 512)
-
-    def test_blocks(self):
-        "Ensure the block checksums are correct."
-        blocks = table(self.rand)
-        self.rand.seek(0)
-        while True:
-            block = self.rand.read(BS)
-            if not block:
-                break
-            match = blocks.get(RollingChecksum(block).digest())
-            self.assertIsNotNone(match)
-            self.assertIn(hashlib.md5(block).hexdigest(), match)
-
-
-class SyncDeltaTestCase(unittest.TestCase):
-    def setUp(self):
-        self.rand1 = StringIO(os.urandom(1024**2))
-        self.rand2 = StringIO(os.urandom(1024**2))
-
-    def test_identical(self):
-        "Ensure two files with ALL matching blocks are handled."
-        blocks = table(self.rand1)
-        ranges, blob = delta(self.rand1, blocks)
-        # No non-matching blocks
-        self.assertEqual(blob.tell(), 0)
-        # All blocks should be collapsed into a single range
-        self.assertEqual(len(ranges), 1)
-
-    def test_different(self):
-        "Ensure two files with NO matching blocks are handled."
-        blocks = table(self.rand1)
-        ranges, blob = delta(self.rand2, blocks)
-        # Blob should contain the entire source file
-        self.assertEqual(blob.tell(), self.rand1.tell())
-        # There should be one range representing the entire source file
-        self.assertEqual(len(ranges), 1)
-
-    def test_mixed(self):
-        "Ensure two files with some overlapping blocks are handled."
-        # Make sure first and last blocks match.
-        matching = [0, 1024]
-        # Pick 510 additional random blocks to make identical (half).
-        for i in xrange(510):
-            while True:
-                block_num = random.randint(0, 1024)
-                if block_num not in matching:
-                    break
-            matching.append(block_num)
-        # Copy our matching blocks from SRC to DST
-        for block_num in matching:
-            self.rand1.seek(block_num*1024)
-            self.rand2.seek(block_num*1024)
-            self.rand2.write(self.rand1.read(1024))
-        # Continue as normal.
-        blocks = table(self.rand1, block_size=1024)
-        ranges, blob = delta(self.rand2, blocks, block_size=1024)
-        self.assertLess(blob.tell(), self.rand1.tell() / 2 + 4096)
-        for i, (direction, offset, length) in enumerate(ranges):
-            if direction == DST:
-                # If the block matches, we should find it's offset / block_size
-                # in matching:
-                block_num = offset / 1024
-                self.assertIn(block_num, matching)
-            # Since blocks are combined, there are not many other assertions we
-            # can make.
-
-
-class SyncPatchTestCase(unittest.TestCase):
-    def setUp(self):
-        self.rand1 = StringIO(os.urandom(1024**2))
-        self.rand2 = StringIO(os.urandom(1024**2))
-
-    def test_1024(self):
-        "Ensure a block_size of 1024 works."
-        blocks = table(self.rand1, block_size=1024)
-        ranges, blob = delta(self.rand2, blocks, block_size=1024)
-        out = patch(self.rand1, ranges, blob)
-        self.assertEqual(hashlib.md5(out.read()).digest(),
-                         hashlib.md5(self.rand2.getvalue()).digest())
-
-    def test_2048(self):
-        "Ensure a block_size of 2048 works."
-        blocks = table(self.rand1, block_size=2048)
-        ranges, blob = delta(self.rand2, blocks, block_size=2048)
-        out = patch(self.rand1, ranges, blob)
-        self.assertEqual(hashlib.md5(out.read()).digest(),
-                         hashlib.md5(self.rand2.getvalue()).digest())
-
-    def test_2001(self):
-        "Ensure an odd block size works."
-        blocks = table(self.rand1, block_size=2001)
-        ranges, blob = delta(self.rand2, blocks, block_size=2001)
-        out = patch(self.rand1, ranges, blob)
-        self.assertEqual(hashlib.md5(out.read()).digest(),
-                         hashlib.md5(self.rand2.getvalue()).digest())
-
-    def test_identical(self):
-        "Ensure patching a file that is identical works."
-        blocks = table(self.rand1, block_size=2048)
-        ranges, blob = delta(self.rand1, blocks, block_size=1024)
-        out = patch(self.rand1, ranges, blob)
-        self.assertEqual(hashlib.md5(out.read()).digest(),
-                         hashlib.md5(self.rand1.getvalue()).digest())
-
-
-# Original functions, I converted to a class and optimized.
-# http://code.activestate.com/recipes/577518-rsync-algorithm/
-def weakchecksum(data):
-    a = b = 0
-    l = len(data)
-    for i in range(l):
-        a += ord(data[i])
-        b += (l - i)*ord(data[i])
-    return (b << 16) | a, a, b
-
-
-def rollingchecksum(removed, new, a, b, blocksize=4096):
-    a -= ord(removed) - ord(new)
-    b -= ord(removed) * blocksize - a
-    return (b << 16) | a, a, b
-
-
-class SyncChecksumTestCase(unittest.TestCase):
-    def test_single(self):
-        "Ensure our checksum on a static buffer is true to the original."
-        for i in xrange(100):
-            data = os.urandom(32)
-            self.assertEqual(RollingChecksum(data).digest(), weakchecksum(data)[0])
-
-    def test_rolling(self):
-        "Ensure our rolling checksum is true to the original."
-        data = deque(os.urandom(32))
-        sum1 = RollingChecksum(data, block_size=32)
-        sum2, a, b = weakchecksum(data)
-        for i in xrange(100):
-            next = os.urandom(1)
-            data.append(next)
-            last = data.popleft()
-            sum1.roll(last, next)
-            sum2, a, b = rollingchecksum(last, next, a, b, blocksize=32)
-            self.assertEqual(sum1.digest(), sum2)
-
-    def test_equality(self):
-        """Ensure that a rolling checksum created on a buffer is equal to the
-        one created when rolling over that buffer."""
-        data = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
-        # Create a checksum of bytes 10 -20
-        sum1 = RollingChecksum(data[10:20], block_size=10)
-        # Create a checksum of bytes 0 - 10
-        sum2 = RollingChecksum(data[:10], block_size=10)
-        # Roll the second checksum over the buffer until reaching bytes 10-20.
-        for i in xrange(10):
-            sum2.roll(data[i], data[10+i])
-        # Sums should now be equal.
-        self.assertEqual(sum1.digest(), sum2.digest())
-
-
 if __name__ == '__main__':
     unittest.main()

From 3e417ed7ac079122903678c2da774c1421f1816e Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Sat, 4 May 2013 09:08:37 -0400
Subject: [PATCH 07/90] Install librsync using apt-get.

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index d8aa259..0c8ef53 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,6 +2,8 @@ language: python
 python:
   - "2.6"
   - "2.7"
+before_install:
+  - sudo apt-get install librsync1 -qq
 install:
   - pip install --timeout=30 pep8 --use-mirrors
   - pip install --timeout=30 https://github.com/dcramer/pyflakes/tarball/master

From 5327374345e38d18f06c3aaa950f1207fee886e0 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Sun, 5 May 2013 15:19:55 -0400
Subject: [PATCH 08/90] Adding coveralls.io Code Coverage.

---
 .coveragerc      |  3 +++
 .coveralls.yml   |  2 ++
 .travis.yml      |  2 ++
 Makefile         |  2 +-
 README.rst       | 12 ++++++++++++
 requirements.txt |  2 ++
 6 files changed, 22 insertions(+), 1 deletion(-)
 create mode 100644 .coveragerc
 create mode 100644 .coveralls.yml

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..0390251
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,3 @@
+[run]
+omit =
+    tests.py
diff --git a/.coveralls.yml b/.coveralls.yml
new file mode 100644
index 0000000..12d5888
--- /dev/null
+++ b/.coveralls.yml
@@ -0,0 +1,2 @@
+repo_token: T89iPkB3rBdrSFoYL6v25QvtEUwNJnhuA
+service_name: travis-ci
diff --git a/.travis.yml b/.travis.yml
index 0c8ef53..2ed9454 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,3 +13,5 @@ before_script:
   - make verify
 script:
   - make test
+after_success:
+  - coveralls
diff --git a/Makefile b/Makefile
index 0488a3f..7e6e4ef 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,5 @@
 test:
-	python tests.py
+	coverage run tests.py
 
 verify:
 	pyflakes -x W smartfile
diff --git a/README.rst b/README.rst
index 2b5ffaa..0a627c3 100644
--- a/README.rst
+++ b/README.rst
@@ -2,6 +2,18 @@
    :alt: Travis CI Status
    :target: https://travis-ci.org/smartfile/client-python
 
+.. image:: https://coveralls.io/repos/smartfile/client-python/badge.png?branch=master
+    :target: https://coveralls.io/r/smartfile/client-python
+    :alt: Code Coverage
+
+.. image:: https://pypip.in/v/smartfile/badge.png
+    :target: https://crate.io/packages/smartfile/
+    :alt: Latest PyPI version
+
+.. image:: https://pypip.in/d/smartfile/badge.png
+    :target: https://crate.io/packages/smartfile/
+    :alt: Number of PyPI downloads
+
 A `SmartFile`_ Open Source project. `Read more`_ about how SmartFile
 uses and contributes to Open Source software.
 
diff --git a/requirements.txt b/requirements.txt
index 127c969..a1563cd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,5 @@ oauthlib
 requests
 requests_oauthlib
 python-librsync
+coveralls
+coverage

From 68c5710a7d732e47897fc361a0521faa1fd7df38 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Sun, 5 May 2013 15:21:04 -0400
Subject: [PATCH 09/90] Use image, not figure.

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 0a627c3..4f6dea0 100644
--- a/README.rst
+++ b/README.rst
@@ -1,4 +1,4 @@
-.. figure:: https://travis-ci.org/smartfile/client-python.png
+.. image:: https://travis-ci.org/smartfile/client-python.png
    :alt: Travis CI Status
    :target: https://travis-ci.org/smartfile/client-python
 

From 179aadbe05d1835ab7c34d9aabc1b572260726d3 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Sun, 5 May 2013 15:27:56 -0400
Subject: [PATCH 10/90] Only report coverage for our code.

---
 .coveragerc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.coveragerc b/.coveragerc
index 0390251..19e5ff4 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,3 +1,3 @@
 [run]
-omit =
-    tests.py
+include = 
+    smartfile/*.py

From 7e70100d5484b7bd5e117198e7ed7a8df02c3f19 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Mon, 6 May 2013 23:30:57 -0400
Subject: [PATCH 11/90] More robust sync client.

---
 smartfile/sync.py | 54 ++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/smartfile/sync.py b/smartfile/sync.py
index a7b60e0..9c6772e 100644
--- a/smartfile/sync.py
+++ b/smartfile/sync.py
@@ -1,4 +1,6 @@
 import os
+import errno
+import tempfile
 
 try:
     import librsync
@@ -21,21 +23,36 @@ class LocalFile(BaseFile):
     perform the steps of the rsync algorithm.
     """
     def signature(self, block_size=None):
+        "Calculates signature for local file."
         kwargs = {}
         if block_size:
             kwargs['block_size'] = block_size
         return librsync.signature(file(self.path, 'rb'), **kwargs)
 
     def delta(self, signature):
+        "Generates delta for local file using remote signature."
         return librsync.delta(file(self.path, 'rb'), signature)
 
     def patch(self, delta):
-        # Open the local file, data may be read from it.
-        f = file(self.path, 'rb')
-        # Unlink the local file, it will remain readable.
-        os.unlink(self.path)
-        # Now patch the file to the local path, replacing it.
-        return librsync.patch(f, delta, o=file(self.path, 'wb'))
+        "Applies remote delta to local file."
+        # Create a temp file in which to store our synced copy. We will handle
+        # deleting it manually, since we may move it instead.
+        with (tempfile.NamedTemporaryFile(prefix='.sync',
+              suffix=os.path.basename(self.path),
+              dir=os.path.dirname(self.path), delete=False)) as output:
+            try:
+                # Open the local file, data may be read from it.
+                with file(self.path, 'rb') as reference:
+                    # Patch the local file into our temporary file.
+                    r = librsync.patch(reference, delta, output)
+                    os.rename(output.name, self.path)
+                    return r
+            finally:
+                try:
+                    os.remove(output.name)
+                except OSError, e:
+                    if e.errno != errno.ENOENT:
+                        raise
 
 
 class RemoteFile(BaseFile):
@@ -48,15 +65,18 @@ def __init__(self, path, api):
         self.api = api
 
     def signature(self, block_size=None):
+        "Requests a signature for remote file via API."
         kwargs = {}
         if block_size:
             kwargs['block_size'] = block_size
         return self.api.get('sync/signature', self.path, **kwargs)
 
     def delta(self, signature):
+        "Generates delta for remote file via API using local file's signature."
         return self.api.post('sync/delta', self.path, signature=signature)
 
     def patch(self, delta):
+        "Applies delta for local file to remote file via API."
         return self.api.post('sync/patch', self.path, delta=delta)
 
 
@@ -65,23 +85,33 @@ class SyncClient(object):
     Synchronizes remote and local files.
     """
     def __init__(self, api, block_size=None):
+        """
+        Synchronizes files with SmartFile using the sync API.
+        """
         self.api = api
         self.block_size = block_size
 
     def sync(self, src, dst):
         """
-        Performs synchronization from source to destination.
+        Performs synchronization from source to destination. Performs the three
+        steps:
+
+        1. Calculate signature of destination.
+        2. Generate delta from source.
+        3. Apply delta to destination.
         """
         return dst.patch(src.delta(dst.signature(block_size=self.block_size)))
 
-    def sync_to_server(self, local, remote):
+    def upload(self, local, remote):
         """
-        Performs synchronization from a local file to a remote file.
+        Performs synchronization from a local file to a remote file. The local
+        path is the source and remote path is the destination.
         """
         self.sync(LocalFile(local), RemoteFile(remote, self.api))
 
-    def sync_from_server(self, local, remote):
+    def download(self, local, remote):
         """
-        Performs synchronization from a remote file to the local system.
+        Performs synchronization from a remote file to a local file. The
+        remote path is the source and the local path is the destination.
         """
-        new = self.sync(RemoteFile(remote, self.api), LocalFile(local))
+        self.sync(RemoteFile(remote, self.api), LocalFile(local))

From 017cb3831eb785bbb79e34993177bfb6be974c71 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 7 May 2013 08:39:25 -0400
Subject: [PATCH 12/90] Documentation for SyncClient.

---
 README.rst | 71 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 70 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 4f6dea0..b496c6a 100644
--- a/README.rst
+++ b/README.rst
@@ -132,7 +132,6 @@ Three methods are supported for providing API credentials using basic authentica
        >>> api = BasicClient(netrcfile='/etc/smartfile.keys')
        >>> api.get('/ping')
 
-
 OAuth Authentication
 --------------------
 
@@ -226,6 +225,9 @@ returned.
     >>> with file('foobar.png', 'wb') as o:
     >>>     shutil.copyfileobj(f, o)
 
+Tasks
+-----
+
 Operations are long-running jobs that are not executed within the time frame
 of an API call. For such operations, a task is created, and the API can be used
 to poll the status of the task.
@@ -240,5 +242,72 @@ to poll the status of the task.
     >>>     if s['status'] == 'SUCCESS':
     >>>         break
 
+Synchronization
+---------------
+
+If you have many files that you wish to keep synchronized between a number of
+computer systems and SmartFile, the sync API can help. The sync API is an
+implementation of the excellent and popular rsync delta algorithm. It is
+completely compatible with the file formats used in librsync version 0.9.7.
+
+The `Rsync algorithm`_ provides a means to synchronize two files by transferring
+just the parts that differ, while retaining the parts that are the same. This
+allows files to be quickly and efficiently synchronized. The rsync algorithm
+is very popular and widely deployed. The implementation in librsync is very
+high quality Open Source software.
+
+SmartFile maintains a `Python wrapper for libarchive`_. The difference between this
+and other wrappers is that the SmartFile wrapper is written using ctypes. Also
+This wrapper is standalone, is specifically written to work with non-disk files
+and has a full test suite.
+
+If you wish to call the synchronization API using the language of your choice,
+you will need to first gain access to librsync. For example, calling librsync
+from Java would require using JNI.
+
+Once you have librsync available, synchronizing files using the SmartFile sync
+API is very simple. The API exposes three calls, corresponding to the three
+steps of the algorithm.
+
+1. Signature (destination)
+2. Delta (source)
+3. Patch (destination)
+
+Depending on the direction of synchronization, source and destination may be
+either your local machine or the SmartFile API. In either case, the steps are
+performed in the same order.
+
+The SmartFile API client provides a simple ``SyncClient`` class that
+demonstrates synchronizing files in either direction. An example of it's usage
+follows.
+
+.. code:: python
+
+    >>> # The sync API uses the same calling conventions as the REST of the API
+    >>> # (pun intended), therefore, we utilize either the Basic or OAuth
+    >>> # flavor of the API client.
+    >>> 
+    >>> from smartfile import BasicClient
+    >>> from smartfile.sync import SyncClient
+    >>> 
+    >>> sync = SyncClient(BasicClient())
+    >>> 
+    >>> # Synchronize TO the server
+    >>> sync.upload('/home/btimby/docs/Resume.pdf', '/docs/Resume.pdf')
+    >>> 
+    >>> # Synchronize FROM the server
+    >>> sync.download('/home/btimby/photos/bricks.jpg', '/photos/bricks.jpg')
+
+The ``SyncClient`` class utilizes libarchive to interact with local files. It uses
+the API client to interact with remote files.
+
+The ``SyncClient`` is not a full synchronization solution, it is only concerned
+with file transfer utilizing deltas. To perform bidirection synchronization (merge
+replication) you would also need to maintain a database of file attributes in
+order to determine if local and remote files are out of sync, which one is
+newest, whether or not the copies conflict and a host of other conditions.
+
 .. _SmartFile: http://www.smartfile.com/
 .. _Read more: http://www.smartfile.com/open-source.html
+.. _Rsync algorithm: http://en.wikipedia.org/wiki/Rsync#Algorithm
+.. _Python wrapper for libarchive: https://www.github.com/smartfile/python-librsync/

From 0cbbe984fbe6a96d09f7b265f93dab57018ed37a Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 7 May 2013 08:55:16 -0400
Subject: [PATCH 13/90] Rearranged headings.

---
 README.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index b496c6a..0dd40bf 100644
--- a/README.rst
+++ b/README.rst
@@ -1,3 +1,9 @@
+.. image:: https://d2xtrvzo9unrru.cloudfront.net/brands/smartfile/logo.png
+   :alt: SmartFile
+
+A `SmartFile`_ Open Source project. `Read more`_ about how SmartFile
+uses and contributes to Open Source software.
+
 .. image:: https://travis-ci.org/smartfile/client-python.png
    :alt: Travis CI Status
    :target: https://travis-ci.org/smartfile/client-python
@@ -14,12 +20,6 @@
     :target: https://crate.io/packages/smartfile/
     :alt: Number of PyPI downloads
 
-A `SmartFile`_ Open Source project. `Read more`_ about how SmartFile
-uses and contributes to Open Source software.
-
-.. figure:: http://www.smartfile.com/images/logo.jpg
-   :alt: SmartFile
-
 Summary
 ------------
 

From f4507145ac7e1fbb324d0773ee8d3c2fb0d614b6 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 7 May 2013 08:58:26 -0400
Subject: [PATCH 14/90] New release.

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index cfca497..9f7444f 100644
--- a/setup.py
+++ b/setup.py
@@ -25,7 +25,7 @@ def get_path(path):
 
 
 name = 'smartfile'
-release = '2'
+release = '3'
 versrel = VERSION + '-' + release
 long_description = file(get_path('README.rst')).read()
 

From b633dadc1dcbc8cde3f58d3307d9ea087c566204 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Mon, 13 May 2013 12:53:58 -0400
Subject: [PATCH 15/90] Sync api lives at /api/__version__/path/sync/ Proxy the
 version attribute of underlying API client.

---
 smartfile/sync.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/smartfile/sync.py b/smartfile/sync.py
index 9c6772e..419f143 100644
--- a/smartfile/sync.py
+++ b/smartfile/sync.py
@@ -69,15 +69,15 @@ def signature(self, block_size=None):
         kwargs = {}
         if block_size:
             kwargs['block_size'] = block_size
-        return self.api.get('sync/signature', self.path, **kwargs)
+        return self.api.get('path/sync/signature', self.path, **kwargs)
 
     def delta(self, signature):
         "Generates delta for remote file via API using local file's signature."
-        return self.api.post('sync/delta', self.path, signature=signature)
+        return self.api.post('path/sync/delta', self.path, signature=signature)
 
     def patch(self, delta):
         "Applies delta for local file to remote file via API."
-        return self.api.post('sync/patch', self.path, delta=delta)
+        return self.api.post('path/sync/patch', self.path, delta=delta)
 
 
 class SyncClient(object):
@@ -91,6 +91,10 @@ def __init__(self, api, block_size=None):
         self.api = api
         self.block_size = block_size
 
+    @property
+    def version(self):
+        return self.api.version
+
     def sync(self, src, dst):
         """
         Performs synchronization from source to destination. Performs the three

From d2b16cdf453743b0cfb6eb8cf6227c14fdf822af Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Mon, 13 May 2013 12:54:39 -0400
Subject: [PATCH 16/90] Mock the sync API server. Test the sync API client.
 General cleanup and enhancement of tests.

---
 tests.py | 239 ++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 165 insertions(+), 74 deletions(-)

diff --git a/tests.py b/tests.py
index 3a9c2c7..633ce40 100644
--- a/tests.py
+++ b/tests.py
@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 
 import os
+import cgi
 import json
 import zlib
 import random
+import base64
 import hashlib
 import urlparse
 import unittest
@@ -19,6 +21,7 @@
 
 from smartfile import BasicClient
 from smartfile import OAuthClient
+from smartfile.sync import SyncClient
 from smartfile.errors import APIError
 from smartfile.errors import RequestError
 
@@ -29,27 +32,41 @@
 ACCESS_TOKEN = 'hIlkipZNmwIJ28HQtQRcbGuXBePQp5'
 ACCESS_SECRET = 'Scen1dwmVtWhjLpJfnilrfdc5OZWCJ'
 
+SYNC_FILE_A = """This is a test file.
+It will be used with the sync client."""
+SYNC_FILE_B = """It will be used with the sync client.
+This is a test file."""
+
+# b64encode(librsync.signature(SYNC_FILE_A)):
+SYNC_SIGNATURE = base64.b64decode('cnMBNgAACAAAAAAIFvwbJw0ItorhbKRo')
+# b64encode(librsync.delta(SYNC_FILE_B, signature)):
+SYNC_DELTA = base64.b64decode('cnMCNkE6SXQgd2lsbCBiZSB1c2VkIHdpdGggdGhlIHN5bmMgY2xpZW50LgpUaGlzIGlzIGEgdGVzdCBmaWxlLgA=')
+
 
 class TestHTTPRequestHandler(BaseHTTPRequestHandler):
     """
     A simple handler that logs requests for examination.
     """
     class TestRequest(object):
-        def __init__(self, method, path, query=None, data=None):
+        def __init__(self, method, path, query=None, data=None, headers=None):
             self.method = method
             self.path = path
             self.query = query
             self.data = data
+            self.headers = headers
 
     def __init__(self, *args, **kwargs):
         self.verbose = kwargs.pop('verbose', False)
         BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def record(self, method, path, query=None, data=None):
-        self.server.requests.append(TestHTTPRequestHandler.TestRequest(method,
-                                    path, query=query, data=data))
+        request = TestHTTPRequestHandler.TestRequest(method, path, query=query,
+                                                     data=data,
+                                                     headers=dict(self.headers.items()))
+        self.server.requests.append(request)
+        return request
 
-    def respond(self):
+    def respond(self, request):
         self.send_response(200)
         self.send_header("Content-type", "text/plain")
         self.end_headers()
@@ -58,11 +75,15 @@ def respond(self):
     def parse_and_record(self, method):
         urlp = urlparse.urlparse(self.path)
         query, data = urlparse.parse_qs(urlp.query), None
-        if method == 'POST':
+        if method in ('POST', 'PUT'):
             l = int(self.headers['Content-Length'])
-            data = urlparse.parse_qs(self.rfile.read(l))
-        self.record(method, urlp.path, query=query, data=data)
-        self.respond()
+            ct, params = cgi.parse_header(self.headers['Content-Type'])
+            if ct == 'multipart/form-data':
+                data = cgi.parse_multipart(self.rfile, params)
+            else:
+                data = urlparse.parse_qs(self.rfile.read(l))
+        request = self.record(method, urlp.path, query=query, data=data)
+        self.respond(request)
 
     def log_message(self, *args, **kwargs):
         if self.verbose:
@@ -88,7 +109,7 @@ class TestHTTPServer(threading.Thread, HTTPServer):
     """
     allow_reuse_address = True
 
-    def __init__(self, address='127.0.0.1', port=0, handler=TestHTTPRequestHandler):
+    def __init__(self, handler, address='127.0.0.1', port=0):
         HTTPServer.__init__(self, (address, port), handler)
         threading.Thread.__init__(self)
         self.requests = []
@@ -103,8 +124,10 @@ class TestServerTestCase(unittest.TestCase):
     """
     Test case that starts our test HTTP server.
     """
+    handler = TestHTTPRequestHandler
+
     def setUp(self):
-        self.server = TestHTTPServer()
+        self.server = TestHTTPServer(self.handler)
 
     def tearDown(self):
         self.server.shutdown()
@@ -117,26 +140,40 @@ def assertRequestCount(self, num=1):
         elif requests < num:
             raise AssertionError('Less than %s request performed' % num)
 
-    def assertMethod(self, method):
+    def assertMethod(self, method, request=-1):
         try:
-            request = self.server.requests[0]
+            request = self.server.requests[request]
         except IndexError:
             raise AssertionError('Cannot assert method without request')
         if request.method != method:
             raise AssertionError('%s is not %s method' % (method,
                                  request.method))
 
-    def assertPath(self, path):
+    def assertPath(self, path, request=-1):
         try:
-            request = self.server.requests[0]
+            request = self.server.requests[request]
         except IndexError:
             raise AssertionError('Cannot assert path without request')
         if request.path != path:
             raise AssertionError('"%s" is not equal to "%s"' % (path,
                                  request.path))
 
+    def assertData(self, key, value, request=-1):
+        try:
+            request = self.server.requests[request]
+        except IndexError:
+            raise AssertionError('Cannot assert data without request')
+        if value not in request.data.get(key, []):
+            raise AssertionError('Request body differs from expectation')
+
+
+class ClientTestCase(TestServerTestCase):
+    def setUp(self):
+        super(ClientTestCase, self).setUp()
+        self.client = self.getClient()
+
 
-class BasicTestCase(TestServerTestCase):
+class BasicTestCase(ClientTestCase):
     def getClient(self, **kwargs):
         kwargs.setdefault('key', API_KEY)
         kwargs.setdefault('password', API_PASSWORD)
@@ -145,7 +182,7 @@ def getClient(self, **kwargs):
         return BasicClient(**kwargs)
 
 
-class OAuthTestCase(TestServerTestCase):
+class OAuthTestCase(ClientTestCase):
     def getClient(self, **kwargs):
         kwargs.setdefault('client_token', CLIENT_TOKEN)
         kwargs.setdefault('client_secret', CLIENT_SECRET)
@@ -159,106 +196,91 @@ def getClient(self, **kwargs):
 class UrlGenerationTestCase(object):
     "Tests that validate 'auto-generated' URLs."
     def test_with_path_id(self):
-        client = self.getClient()
-        client.get('/path/data', '/the/file/path')
+        self.client.get('/path/data', '/the/file/path')
         self.assertMethod('GET')
         self.assertPath('/api/{0}/path/data/the/file/path/'.format(
-            client.version))
+            self.client.version))
 
     def test_with_int_id(self):
-        client = self.getClient()
-        client.get('/access/user', 42)
+        self.client.get('/access/user', 42)
         self.assertMethod('GET')
-        self.assertPath('/api/{0}/access/user/42/'.format(client.version))
+        self.assertPath('/api/{0}/access/user/42/'.format(self.client.version))
 
     def test_with_version(self):
-        client = self.getClient(version='3.1')
-        client.get('/ping')
-        self.assertMethod('GET')
-        self.assertPath('/api/{0}/ping/'.format(client.version))
+        for major in xrange(10):
+            for minor in xrange(10):
+                client = self.getClient(version='%s.%s' % (major, minor))
+                client.get('/ping')
+                self.assertMethod('GET')
+                self.assertPath('/api/{0}/ping/'.format(client.version))
 
 
 class MethodTestCase(object):
     "Tests the HTTP methods used by CRUD methods."
     def test_call_is_GET(self):
-        client = self.getClient()
-        client('/user', 'bobafett')
+        self.client('/user', 'bobafett')
         self.assertMethod('GET')
 
     def test_post_is_POST(self):
-        client = self.getClient()
-        client.post('/user', username='bobafett', email='bobafett@example.com')
+        self.client.post('/user', username='bobafett', email='bobafett@example.com')
         self.assertMethod('POST')
 
     def test_get_is_GET(self):
-        client = self.getClient()
-        client.get('/user', 'bobafett')
+        self.client.get('/user', 'bobafett')
         self.assertMethod('GET')
 
     def test_put_is_PUT(self):
-        client = self.getClient()
-        client.put('/user', 'bobafett', full_name='Boba Fett')
+        self.client.put('/user', 'bobafett', full_name='Boba Fett')
         self.assertMethod('PUT')
 
     def test_delete_is_DELETE(self):
-        client = self.getClient()
-        client.delete('/user', 'bobafett')
+        self.client.delete('/user', 'bobafett')
         self.assertMethod('DELETE')
 
 
 class DownloadTestCase(object):
     def test_file_response(self):
-        client = self.getClient()
-        r = client.get('/user')
+        r = self.client.get('/user')
         self.assertTrue(hasattr(r, 'read'), 'File-like object not returned.')
         self.assertEqual(r.read(), 'Hello World!')
 
 
 class UploadTestCase(object):
     def test_file_upload(self):
-        client = self.getClient()
         fd, t = tempfile.mkstemp()
         os.close(fd)
         try:
-            client.post('/path/data', 'foobar.png', file=file(t))
-        except Exception, e:
-            self.fail('POSTing a file failed. %s' % e)
+            self.client.post('/path/data', 'foobar.png', file=file(t))
         finally:
-            try:
-                os.unlink(t)
-            except:
-                pass
+            os.unlink(t)
 
 
-class BasicEnvironTestCase(BasicTestCase):
+class BasicEnvironTestCase(UrlGenerationTestCase, BasicTestCase):
     "Tests that the API client reads settings from ENV."
     def setUp(self):
-        super(BasicEnvironTestCase, self).setUp()
         os.environ['SMARTFILE_API_KEY'] = API_KEY
         os.environ['SMARTFILE_API_PASSWORD'] = API_KEY
+        super(BasicEnvironTestCase, self).setUp()
 
     def tearDown(self):
         super(BasicEnvironTestCase, self).tearDown()
         del os.environ['SMARTFILE_API_KEY']
         del os.environ['SMARTFILE_API_PASSWORD']
 
-    def test_read_from_env(self):
-        # Blank out the credentials, the client should read them from the
-        # environment variables.
-        client = self.getClient(key=None, password=None)
-        client.get('/ping')
-        self.assertMethod('GET')
-        self.assertPath('/api/{0}/ping/'.format(client.version))
+    def getClient(self, **kwargs):
+        kwargs['key'] = None
+        kwargs['password'] = None
+        return super(BasicEnvironTestCase, self).getClient(**kwargs)
 
 
-class OAuthEnvironTestCase(OAuthTestCase):
+class OAuthEnvironTestCase(UrlGenerationTestCase, OAuthTestCase):
     "Tests that the API client reads settings from ENV."
     def setUp(self):
-        super(OAuthEnvironTestCase, self).setUp()
         os.environ['SMARTFILE_CLIENT_TOKEN'] = CLIENT_TOKEN
         os.environ['SMARTFILE_CLIENT_SECRET'] = CLIENT_SECRET
         os.environ['SMARTFILE_ACCESS_TOKEN'] = ACCESS_TOKEN
         os.environ['SMARTFILE_ACCESS_SECRET'] = ACCESS_SECRET
+        super(OAuthEnvironTestCase, self).setUp()
 
     def tearDown(self):
         super(OAuthEnvironTestCase, self).tearDown()
@@ -267,13 +289,10 @@ def tearDown(self):
         del os.environ['SMARTFILE_ACCESS_TOKEN']
         del os.environ['SMARTFILE_ACCESS_SECRET']
 
-    def test_read_from_env(self):
-        # Blank out the credentials, the client should read them from the
-        # environment variables.
-        client = self.getClient(client_token=None, client_secret=None)
-        client.get('/ping')
-        self.assertMethod('GET')
-        self.assertPath('/api/{0}/ping/'.format(client.version))
+    def getClient(self, **kwargs):
+        kwargs['client_token'] = None
+        kwargs['client_secret'] = None
+        return super(OAuthEnvironTestCase, self).getClient(**kwargs)
 
 
 class BasicClientTestCase(DownloadTestCase, UploadTestCase, MethodTestCase,
@@ -317,7 +336,7 @@ def test_blank_access_token(self):
 
 
 class HTTPThrottleRequestHandler(TestHTTPRequestHandler):
-    def respond(self):
+    def respond(self, request):
         self.send_response(503)
         self.send_header("X-Throttle", "throttled; next=0.01 sec")
         self.end_headers()
@@ -325,12 +344,10 @@ def respond(self):
 
 
 class ThrottleTestCase(object):
-    def setUp(self):
-        self.server = TestHTTPServer(handler=HTTPThrottleRequestHandler)
+    handler = HTTPThrottleRequestHandler
 
     def test_throttle_GET(self):
-        client = self.getClient()
-        self.assertRaises(RequestError, client.get, '/ping')
+        self.assertRaises(RequestError, self.client.get, '/ping')
         self.assertRequestCount(3)
 
 
@@ -343,7 +360,7 @@ class OAuthThrottleTestCase(ThrottleTestCase, OAuthTestCase):
 
 
 class HTTPJSONRequestHandler(TestHTTPRequestHandler):
-    def respond(self):
+    def respond(self, request):
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
@@ -351,12 +368,10 @@ def respond(self):
 
 
 class JSONTestCase(object):
-    def setUp(self):
-        self.server = TestHTTPServer(handler=HTTPJSONRequestHandler)
+    handler = HTTPJSONRequestHandler
 
     def test_throttle_GET(self):
-        client = self.getClient()
-        r = client.get('/user')
+        r = self.client.get('/user')
         self.assertMethod('GET')
         self.assertEqual(r, { 'foo': 'bar' })
 
@@ -369,6 +384,82 @@ class OAuthJSONTestCase(JSONTestCase, OAuthTestCase):
     pass
 
 
+class SyncRequestHandler(TestHTTPRequestHandler):
+    def respond(self, request):
+        if '/signature/' in request.path:
+            return self.respond_signature()
+        elif '/delta/' in request.path:
+            return self.respond_delta()
+        elif '/patch/' in request.path:
+            return self.respond_patch()
+        else:
+            return super(SyncRequestHandler, self).respond()
+
+    def respond_signature(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/librsync-signature")
+        self.end_headers()
+        self.wfile.write(SYNC_SIGNATURE)
+
+    def respond_delta(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/librsync-delta")
+        self.end_headers()
+        self.wfile.write(SYNC_DELTA)
+
+    def respond_patch(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/json")
+        self.end_headers()
+        self.wfile.write(json.dumps({'foo': 'bar'}))
+
+
+class SyncTestCase(object):
+    "Test delta transfer via sync API."
+    handler = SyncRequestHandler
+
+    def getClient(self, **kwargs):
+        "Override to return a sync client that uses the underlying client."
+        client = super(SyncTestCase, self).getClient(**kwargs)
+        return SyncClient(client)
+
+    def test_upload(self):
+        "Ensure we can upload via sync API."
+        fd, t = tempfile.mkstemp()
+        os.write(fd, SYNC_FILE_B)
+        os.close(fd)
+        try:
+            self.client.upload(t, '/unittest/sync')
+        finally:
+            os.unlink(t)
+        self.assertRequestCount(2)
+        self.assertPath('/api/%s/path/sync/signature/unittest/sync/' % self.client.version, request=0)
+        self.assertPath('/api/%s/path/sync/patch/unittest/sync/' % self.client.version, request=1)
+        self.assertData('delta', SYNC_DELTA, request=1)
+
+    def test_download(self):
+        "Ensure we can download via sync API."
+        fd, t = tempfile.mkstemp()
+        os.write(fd, SYNC_FILE_A)
+        os.close(fd)
+        try:
+            self.client.download(t, '/unittest/sync')
+            # The local file contents should have been changed.
+            self.assertEqual(SYNC_FILE_B, file(t).read())
+        finally:
+            os.unlink(t)
+        self.assertRequestCount(1)
+        self.assertPath('/api/%s/path/sync/delta/unittest/sync/' % self.client.version)
+
+
+class BasicSyncTestCase(SyncTestCase, BasicTestCase):
+    pass
+
+
+class OAuthSyncTestCase(SyncTestCase, OAuthTestCase):
+    pass
+
+
 # TODO: Test with missing oauthlib...
 # Must invoke an ImportError when smartfile tries to import it. Then the test
 # case should verify that the correct exception (NotImplementedError) is raised

From 43025c62163d42a776c8b5f0458a87ea0ae379ee Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 4 Jun 2013 03:04:30 -0400
Subject: [PATCH 17/90] Ported to python 3.

---
 .travis.yml           |  3 +-
 Makefile              | 10 +++++--
 requirements.txt      |  1 +
 setup.py              | 11 +++----
 smartfile/__init__.py | 30 +++++++++++--------
 smartfile/errors.py   |  5 ++--
 smartfile/sync.py     |  8 ++---
 tests.py              | 69 ++++++++++++++++++++++---------------------
 8 files changed, 77 insertions(+), 60 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2ed9454..b55be14 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -2,11 +2,12 @@ language: python
 python:
   - "2.6"
   - "2.7"
+  - "3.2"
 before_install:
   - sudo apt-get install librsync1 -qq
 install:
   - pip install --timeout=30 pep8 --use-mirrors
-  - pip install --timeout=30 https://github.com/dcramer/pyflakes/tarball/master
+  - pip install --timeout=30 pyflakes --use-mirrors
   - pip install --timeout=30 -r requirements.txt --use-mirrors
   - pip install --timeout=30 -q -e . --use-mirrors
 before_script:
diff --git a/Makefile b/Makefile
index 7e6e4ef..f53fc73 100644
--- a/Makefile
+++ b/Makefile
@@ -2,8 +2,8 @@ test:
 	coverage run tests.py
 
 verify:
-	pyflakes -x W smartfile
-	pep8 --exclude=migrations --ignore=E501,E225 smartfile
+	pyflakes smartfile
+	pep8 --ignore=E501,E225 smartfile
 
 install:
 	python setup.py install
@@ -15,3 +15,9 @@ publish:
 profile:
 	python profile.py
 
+clean:
+	find . -name *.pyc -delete
+
+distclean: clean
+	rm -rf env
+
diff --git a/requirements.txt b/requirements.txt
index a1563cd..3faf34c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+six
 oauthlib
 requests
 requests_oauthlib
diff --git a/setup.py b/setup.py
index 9f7444f..3c4ed43 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,8 @@
 
 import os
 import re
-from distutils.core import setup
+#from distutils.core import setup
+from setuptools import setup
 
 VERSION_PATTERN = re.compile(r'^[^#]*__version__\W*\=\W*["\'](.*)["\']')
 VERSION = None
@@ -12,7 +13,7 @@ def get_path(path):
     return os.path.join(os.path.dirname(__file__), path)
 
 
-with file(get_path('smartfile/__init__.py')) as f:
+with open(get_path('smartfile/__init__.py'), 'rb') as f:
     for line in f.xreadlines():
         m = VERSION_PATTERN.search(line)
         if m:
@@ -25,9 +26,9 @@ def get_path(path):
 
 
 name = 'smartfile'
-release = '3'
+release = '6'
 versrel = VERSION + '-' + release
-long_description = file(get_path('README.rst')).read()
+long_description = open(get_path('README.rst'), 'rb').read()
 
 
 setup(
@@ -35,7 +36,7 @@ def get_path(path):
     version=versrel,
     description='A Python client for the SmartFile API.',
     long_description=long_description,
-    requires=[
+    install_requires=[
         'oauthlib',
         'requests',
         'requests_oauthlib',
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 1a1ca17..20383d5 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -1,9 +1,13 @@
 import re
 import os
+import six
 import time
 import string
 import urllib
-import urlparse
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
 import requests
 
 from netrc import netrc
@@ -26,12 +30,12 @@
 def clean_tokens(*args):
     if not all(map(bool, args)):
         raise ValueError("not provided")
-    args = map(string.strip, args)
+    args = list(map(lambda x: x.strip(), args))
     for i, arg in enumerate(args):
         if len(arg) < 30:
             raise ValueError("too short")
-        if not isinstance(arg, unicode):
-            arg = unicode(arg)
+        if not isinstance(arg, six.text_type):
+            arg = six.u(arg)
         args[i] = arg
     return args
 
@@ -47,7 +51,7 @@ def _do_request(self, request, url, **kwargs):
         "Actually makes the HTTP request."
         try:
             response = request(url, stream=True, **kwargs)
-        except RequestException, e:
+        except RequestException as e:
             raise RequestError(e)
         else:
             if response.status_code >= 400:
@@ -57,7 +61,7 @@ def _do_request(self, request, url, **kwargs):
             try:
                 # Try to decode as JSON
                 return response.json()
-            except ValueError:
+            except (TypeError, ValueError):
                 # If that fails, return the text.
                 return response.text
         else:
@@ -73,7 +77,7 @@ def _request(self, method, endpoint, id=None, **kwargs):
         data = kwargs.get('data')
         if data:
             files = {}
-            for name, value in data.items():
+            for name, value in list(data.items()):
                 # Value might be a file-like object (with a read method), or it
                 # might be a (filename, file-like) tuple.
                 if hasattr(value, 'read') or isinstance(value, tuple):
@@ -101,7 +105,7 @@ def _request(self, method, endpoint, id=None, **kwargs):
             trys += 1
             try:
                 return self._do_request(request, url, **kwargs)
-            except ResponseError, e:
+            except ResponseError as e:
                 if self.throttle_wait and e.status_code == 503:
                     m = THROTTLE_PATTERN.match(e.response.headers.get('x-throttle', ''))
                     if m:
@@ -174,8 +178,8 @@ def _do_request(self, *args, **kwargs):
     class OAuthToken(object):
         "Internal representation of an OAuth (token, secret) tuple."
         def __init__(self, token=None, secret=None):
-            self.token = token and unicode(token)
-            self.secret = secret and unicode(secret)
+            self.token = token and six.u(token)
+            self.secret = secret and six.u(secret)
 
         def __iter__(self):
             yield self.token
@@ -227,7 +231,7 @@ def _do_request(self, *args, **kwargs):
         def get_request_token(self, callback=None):
             "The first step of the OAuth workflow."
             if callback:
-                callback = unicode(callback)
+                callback = six.u(callback)
             oauth = OAuth1(self._client.token,
                            client_secret=self._client.secret,
                            callback_uri=callback,
@@ -253,7 +257,7 @@ def get_access_token(self, request=None, verifier=None):
             """The final step of the OAuth workflow. After this the client can make
             API calls."""
             if verifier:
-                verifier = unicode(verifier)
+                verifier = six.u(verifier)
             if request is None:
                 if not self.__request.is_valid():
                     raise APIError('You must obtain a request token to request '
@@ -264,7 +268,7 @@ def get_access_token(self, request=None, verifier=None):
                            client_secret=self._client.secret,
                            resource_owner_key=request.token,
                            resource_owner_secret=request.secret,
-                           verifier=unicode(verifier),
+                           verifier=six.u(verifier),
                            signature_method=SIGNATURE_PLAINTEXT)
             r = requests.post(urlparse.urljoin(self.url, 'oauth/access_token/'), auth=oauth)
             credentials = urlparse.parse_qs(r.text)
diff --git a/smartfile/errors.py b/smartfile/errors.py
index f6aea4e..758844a 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -1,3 +1,4 @@
+import six
 
 
 class APIError(Exception):
@@ -25,9 +26,9 @@ def __init__(self, response, *args, **kwargs):
             json = response.json()
         except ValueError:
             if self.status_code == 404:
-                self.detail = u'Invalid URL, check your API path'
+                self.detail = six.u('Invalid URL, check your API path')
             else:
-                self.detail = u'Server error; check response for errors'
+                self.detail = six.u('Server error; check response for errors')
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']
diff --git a/smartfile/sync.py b/smartfile/sync.py
index 419f143..861711a 100644
--- a/smartfile/sync.py
+++ b/smartfile/sync.py
@@ -27,11 +27,11 @@ def signature(self, block_size=None):
         kwargs = {}
         if block_size:
             kwargs['block_size'] = block_size
-        return librsync.signature(file(self.path, 'rb'), **kwargs)
+        return librsync.signature(open(self.path, 'rb'), **kwargs)
 
     def delta(self, signature):
         "Generates delta for local file using remote signature."
-        return librsync.delta(file(self.path, 'rb'), signature)
+        return librsync.delta(open(self.path, 'rb'), signature)
 
     def patch(self, delta):
         "Applies remote delta to local file."
@@ -42,7 +42,7 @@ def patch(self, delta):
               dir=os.path.dirname(self.path), delete=False)) as output:
             try:
                 # Open the local file, data may be read from it.
-                with file(self.path, 'rb') as reference:
+                with open(self.path, 'rb') as reference:
                     # Patch the local file into our temporary file.
                     r = librsync.patch(reference, delta, output)
                     os.rename(output.name, self.path)
@@ -50,7 +50,7 @@ def patch(self, delta):
             finally:
                 try:
                     os.remove(output.name)
-                except OSError, e:
+                except OSError as e:
                     if e.errno != errno.ENOENT:
                         raise
 
diff --git a/tests.py b/tests.py
index 633ce40..ec3f691 100644
--- a/tests.py
+++ b/tests.py
@@ -1,23 +1,29 @@
-# -*- coding: utf-8 -*-
-
 import os
+import six
 import cgi
 import json
 import zlib
 import random
 import base64
 import hashlib
-import urlparse
+try:
+    import urlparse
+except ImportError:
+    import urllib.parse as urlparse
 import unittest
 import tempfile
 import threading
 
 from collections import deque
 
-from StringIO import StringIO
+from six import StringIO
 
-from BaseHTTPServer import HTTPServer
-from BaseHTTPServer import BaseHTTPRequestHandler
+try:
+    from BaseHTTPServer import HTTPServer
+    from BaseHTTPServer import BaseHTTPRequestHandler
+except ImportError:
+    from http.server import HTTPServer
+    from http.server import BaseHTTPRequestHandler
 
 from smartfile import BasicClient
 from smartfile import OAuthClient
@@ -25,6 +31,7 @@
 from smartfile.errors import APIError
 from smartfile.errors import RequestError
 
+
 API_KEY = '8g1aq1UF2QfZTG47yEVhVLAFqyfDdp'
 API_PASSWORD = '3II3UFD3pBAwy3Rbz8mVWBhJTA2Gvd'
 CLIENT_TOKEN = '8oWot4KrppJDzfokDsHNJrND0Ay13s'
@@ -32,15 +39,15 @@
 ACCESS_TOKEN = 'hIlkipZNmwIJ28HQtQRcbGuXBePQp5'
 ACCESS_SECRET = 'Scen1dwmVtWhjLpJfnilrfdc5OZWCJ'
 
-SYNC_FILE_A = """This is a test file.
-It will be used with the sync client."""
-SYNC_FILE_B = """It will be used with the sync client.
-This is a test file."""
+SYNC_FILE_A = six.b("""This is a test file.
+It will be used with the sync client.""")
+SYNC_FILE_B = six.b("""It will be used with the sync client.
+This is a test file.""")
 
 # b64encode(librsync.signature(SYNC_FILE_A)):
-SYNC_SIGNATURE = base64.b64decode('cnMBNgAACAAAAAAIFvwbJw0ItorhbKRo')
+SYNC_SIGNATURE = base64.b64decode(six.b('cnMBNgAACAAAAAAIFvwbJw0ItorhbKRo'))
 # b64encode(librsync.delta(SYNC_FILE_B, signature)):
-SYNC_DELTA = base64.b64decode('cnMCNkE6SXQgd2lsbCBiZSB1c2VkIHdpdGggdGhlIHN5bmMgY2xpZW50LgpUaGlzIGlzIGEgdGVzdCBmaWxlLgA=')
+SYNC_DELTA = base64.b64decode(six.b('cnMCNkE6SXQgd2lsbCBiZSB1c2VkIHdpdGggdGhlIHN5bmMgY2xpZW50LgpUaGlzIGlzIGEgdGVzdCBmaWxlLgA='))
 
 
 class TestHTTPRequestHandler(BaseHTTPRequestHandler):
@@ -49,7 +56,7 @@ class TestHTTPRequestHandler(BaseHTTPRequestHandler):
     """
     class TestRequest(object):
         def __init__(self, method, path, query=None, data=None, headers=None):
-            self.method = method
+            self.method = method.upper()
             self.path = path
             self.query = query
             self.data = data
@@ -70,7 +77,7 @@ def respond(self, request):
         self.send_response(200)
         self.send_header("Content-type", "text/plain")
         self.end_headers()
-        self.wfile.write("Hello World!")
+        self.wfile.write(six.b("Hello World!"))
 
     def parse_and_record(self, method):
         urlp = urlparse.urlparse(self.path)
@@ -79,7 +86,7 @@ def parse_and_record(self, method):
             l = int(self.headers['Content-Length'])
             ct, params = cgi.parse_header(self.headers['Content-Type'])
             if ct == 'multipart/form-data':
-                data = cgi.parse_multipart(self.rfile, params)
+                data = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'})
             else:
                 data = urlparse.parse_qs(self.rfile.read(l))
         request = self.record(method, urlp.path, query=query, data=data)
@@ -145,26 +152,21 @@ def assertMethod(self, method, request=-1):
             request = self.server.requests[request]
         except IndexError:
             raise AssertionError('Cannot assert method without request')
-        if request.method != method:
-            raise AssertionError('%s is not %s method' % (method,
-                                 request.method))
+        self.assertEqual(method.upper(), request.method)
 
     def assertPath(self, path, request=-1):
         try:
             request = self.server.requests[request]
         except IndexError:
             raise AssertionError('Cannot assert path without request')
-        if request.path != path:
-            raise AssertionError('"%s" is not equal to "%s"' % (path,
-                                 request.path))
+        self.assertEqual(path, request.path)
 
     def assertData(self, key, value, request=-1):
         try:
             request = self.server.requests[request]
         except IndexError:
             raise AssertionError('Cannot assert data without request')
-        if value not in request.data.get(key, []):
-            raise AssertionError('Request body differs from expectation')
+        self.assertIn(value, request.data.getvalue(key, []))
 
 
 class ClientTestCase(TestServerTestCase):
@@ -207,8 +209,8 @@ def test_with_int_id(self):
         self.assertPath('/api/{0}/access/user/42/'.format(self.client.version))
 
     def test_with_version(self):
-        for major in xrange(10):
-            for minor in xrange(10):
+        for major in range(10):
+            for minor in range(10):
                 client = self.getClient(version='%s.%s' % (major, minor))
                 client.get('/ping')
                 self.assertMethod('GET')
@@ -242,7 +244,7 @@ class DownloadTestCase(object):
     def test_file_response(self):
         r = self.client.get('/user')
         self.assertTrue(hasattr(r, 'read'), 'File-like object not returned.')
-        self.assertEqual(r.read(), 'Hello World!')
+        self.assertEqual(r.read(), six.b('Hello World!'))
 
 
 class UploadTestCase(object):
@@ -250,7 +252,7 @@ def test_file_upload(self):
         fd, t = tempfile.mkstemp()
         os.close(fd)
         try:
-            self.client.post('/path/data', 'foobar.png', file=file(t))
+            self.client.post('/path/data', 'foobar.png', file=open(t, 'rb'))
         finally:
             os.unlink(t)
 
@@ -309,8 +311,8 @@ def test_netrc(self):
                     address, port = address
                 else:
                     port = self.server.server_port
-                netrc = "machine 127.0.0.1:%s\n  login %s\n  password %s" % (
-                        port, API_KEY, API_PASSWORD)
+                netrc = six.b("machine 127.0.0.1:%s\n  login %s\n  password %s" % (
+                        port, API_KEY, API_PASSWORD))
                 os.write(fd, netrc)
             finally:
                 os.close(fd)
@@ -340,7 +342,7 @@ def respond(self, request):
         self.send_response(503)
         self.send_header("X-Throttle", "throttled; next=0.01 sec")
         self.end_headers()
-        self.wfile.write("Request Throttled!")
+        self.wfile.write(six.b("Request Throttled!"))
 
 
 class ThrottleTestCase(object):
@@ -364,7 +366,7 @@ def respond(self, request):
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
-        self.wfile.write(json.dumps({ 'foo': 'bar' }))
+        self.wfile.write(six.b(json.dumps({ 'foo': 'bar' })))
 
 
 class JSONTestCase(object):
@@ -411,7 +413,7 @@ def respond_patch(self):
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
-        self.wfile.write(json.dumps({'foo': 'bar'}))
+        self.wfile.write(six.b(json.dumps({'foo': 'bar'})))
 
 
 class SyncTestCase(object):
@@ -445,11 +447,12 @@ def test_download(self):
         try:
             self.client.download(t, '/unittest/sync')
             # The local file contents should have been changed.
-            self.assertEqual(SYNC_FILE_B, file(t).read())
+            self.assertEqual(SYNC_FILE_B, open(t, 'rb').read())
         finally:
             os.unlink(t)
         self.assertRequestCount(1)
         self.assertPath('/api/%s/path/sync/delta/unittest/sync/' % self.client.version)
+        self.assertData('signature', SYNC_SIGNATURE)
 
 
 class BasicSyncTestCase(SyncTestCase, BasicTestCase):

From ffc5c691c4e1cca0249893b255de9f6dfce9f5bf Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 4 Jun 2013 03:08:06 -0400
Subject: [PATCH 18/90] New release.

---
 setup.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index 3c4ed43..8323b5f 100644
--- a/setup.py
+++ b/setup.py
@@ -13,8 +13,8 @@ def get_path(path):
     return os.path.join(os.path.dirname(__file__), path)
 
 
-with open(get_path('smartfile/__init__.py'), 'rb') as f:
-    for line in f.xreadlines():
+with open(get_path('smartfile/__init__.py'), 'r') as f:
+    for line in f.readlines():
         m = VERSION_PATTERN.search(line)
         if m:
             VERSION = m.group(1)
@@ -26,9 +26,9 @@ def get_path(path):
 
 
 name = 'smartfile'
-release = '6'
+release = '7'
 versrel = VERSION + '-' + release
-long_description = open(get_path('README.rst'), 'rb').read()
+long_description = open(get_path('README.rst'), 'r').read()
 
 
 setup(

From 3c5e005d28f5a65b49889ea06364973d591a9025 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 4 Jun 2013 03:12:46 -0400
Subject: [PATCH 19/90] Fix pyflakes warning.

---
 smartfile/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 20383d5..3218a4d 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -2,13 +2,14 @@
 import os
 import six
 import time
-import string
 import urllib
+import requests
 try:
     import urlparse
+    # Fixed pyflakes warning...
+    urlparse
 except ImportError:
-    import urllib.parse as urlparse
-import requests
+    from urllib import parse as urlparse
 
 from netrc import netrc
 

From 35303c700099e3b3db3b9b06a7337df0f435150f Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@gmail.com>
Date: Tue, 4 Jun 2013 03:17:31 -0400
Subject: [PATCH 20/90] Python 2.6 compat.

---
 tests.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests.py b/tests.py
index ec3f691..2feedbe 100644
--- a/tests.py
+++ b/tests.py
@@ -168,6 +168,10 @@ def assertData(self, key, value, request=-1):
             raise AssertionError('Cannot assert data without request')
         self.assertIn(value, request.data.getvalue(key, []))
 
+    def assertIn(self, test_value, expected_set):
+        msg = "%s did not occur in %s" % (test_value, expected_set)
+        self.assert_(test_value in expected_set, msg) 
+
 
 class ClientTestCase(TestServerTestCase):
     def setUp(self):

From 6a949735c3da2c1c33951fd20f2b0c6b0f95b6ad Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@umail.iu.edu>
Date: Thu, 28 Aug 2014 16:34:41 -0400
Subject: [PATCH 21/90] pep8 fixes

---
 smartfile/__init__.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 3218a4d..fb3ccec 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -172,9 +172,7 @@ def _do_request(self, *args, **kwargs):
     from requests_oauthlib import OAuth1
     from oauthlib.oauth1 import SIGNATURE_PLAINTEXT
 
-    #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
-    #                OAuth, if available.
-    #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
+    # OAuth, if available.
 
     class OAuthToken(object):
         "Internal representation of an OAuth (token, secret) tuple."
@@ -279,9 +277,7 @@ def get_access_token(self, request=None, verifier=None):
 
 
 except ImportError:
-    #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
-    #              OAuth, if not available.
-    #*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~*~
+    # OAuth, if not available.
 
     # Instead of a class, define this as a function, thus when a user tries to
     # "instantiate" it, they receive an exception.

From a945d0903de66e7ad9995619d44a255be54ef9e7 Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@umail.iu.edu>
Date: Mon, 6 Oct 2014 10:48:26 -0400
Subject: [PATCH 22/90] Update README.rst

Removed information regarding Rsync
---
 README.rst | 66 ------------------------------------------------------
 1 file changed, 66 deletions(-)

diff --git a/README.rst b/README.rst
index 0dd40bf..61c2509 100644
--- a/README.rst
+++ b/README.rst
@@ -242,72 +242,6 @@ to poll the status of the task.
     >>>     if s['status'] == 'SUCCESS':
     >>>         break
 
-Synchronization
----------------
-
-If you have many files that you wish to keep synchronized between a number of
-computer systems and SmartFile, the sync API can help. The sync API is an
-implementation of the excellent and popular rsync delta algorithm. It is
-completely compatible with the file formats used in librsync version 0.9.7.
-
-The `Rsync algorithm`_ provides a means to synchronize two files by transferring
-just the parts that differ, while retaining the parts that are the same. This
-allows files to be quickly and efficiently synchronized. The rsync algorithm
-is very popular and widely deployed. The implementation in librsync is very
-high quality Open Source software.
-
-SmartFile maintains a `Python wrapper for libarchive`_. The difference between this
-and other wrappers is that the SmartFile wrapper is written using ctypes. Also
-This wrapper is standalone, is specifically written to work with non-disk files
-and has a full test suite.
-
-If you wish to call the synchronization API using the language of your choice,
-you will need to first gain access to librsync. For example, calling librsync
-from Java would require using JNI.
-
-Once you have librsync available, synchronizing files using the SmartFile sync
-API is very simple. The API exposes three calls, corresponding to the three
-steps of the algorithm.
-
-1. Signature (destination)
-2. Delta (source)
-3. Patch (destination)
-
-Depending on the direction of synchronization, source and destination may be
-either your local machine or the SmartFile API. In either case, the steps are
-performed in the same order.
-
-The SmartFile API client provides a simple ``SyncClient`` class that
-demonstrates synchronizing files in either direction. An example of it's usage
-follows.
-
-.. code:: python
-
-    >>> # The sync API uses the same calling conventions as the REST of the API
-    >>> # (pun intended), therefore, we utilize either the Basic or OAuth
-    >>> # flavor of the API client.
-    >>> 
-    >>> from smartfile import BasicClient
-    >>> from smartfile.sync import SyncClient
-    >>> 
-    >>> sync = SyncClient(BasicClient())
-    >>> 
-    >>> # Synchronize TO the server
-    >>> sync.upload('/home/btimby/docs/Resume.pdf', '/docs/Resume.pdf')
-    >>> 
-    >>> # Synchronize FROM the server
-    >>> sync.download('/home/btimby/photos/bricks.jpg', '/photos/bricks.jpg')
-
-The ``SyncClient`` class utilizes libarchive to interact with local files. It uses
-the API client to interact with remote files.
-
-The ``SyncClient`` is not a full synchronization solution, it is only concerned
-with file transfer utilizing deltas. To perform bidirection synchronization (merge
-replication) you would also need to maintain a database of file attributes in
-order to determine if local and remote files are out of sync, which one is
-newest, whether or not the copies conflict and a host of other conditions.
 
 .. _SmartFile: http://www.smartfile.com/
 .. _Read more: http://www.smartfile.com/open-source.html
-.. _Rsync algorithm: http://en.wikipedia.org/wiki/Rsync#Algorithm
-.. _Python wrapper for libarchive: https://www.github.com/smartfile/python-librsync/

From b81d017b36eeed69e55b9c7536f53f79de0973ac Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@umail.iu.edu>
Date: Thu, 29 Jan 2015 18:34:21 -0500
Subject: [PATCH 23/90] Update .travis.yml

---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index b55be14..09b9332 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,3 +16,5 @@ script:
   - make test
 after_success:
   - coveralls
+notifications:
+  slack: smartfile:tbDIPzVJIPBpSz29kQw6b8RQ

From 245cace665fed215fa00909877d6c2988cf2e0fb Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@iupui.edu>
Date: Mon, 23 Feb 2015 17:34:34 -0500
Subject: [PATCH 24/90] Removed 'six' dependency.

---
 requirements.txt      |  1 -
 setup.py              |  4 ++--
 smartfile/__init__.py | 19 ++++++-------------
 3 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 3faf34c..a1563cd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
-six
 oauthlib
 requests
 requests_oauthlib
diff --git a/setup.py b/setup.py
index 8323b5f..502523e 100644
--- a/setup.py
+++ b/setup.py
@@ -42,9 +42,9 @@ def get_path(path):
         'requests_oauthlib',
     ],
     author='SmartFile',
-    author_email='info@smartfile.com',
+    author_email='tech@smartfile.com',
     maintainer='Ben Timby',
-    maintainer_email='btimby@gmail.com',
+    maintainer_email='tech@smartfile.com',
     url='http://github.com/smartfile/client-python/',
     license='MIT',
     packages=['smartfile'],
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index fb3ccec..c929179 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -1,9 +1,8 @@
-import re
+from netrc import netrc
 import os
-import six
+import re
 import time
 import urllib
-import requests
 try:
     import urlparse
     # Fixed pyflakes warning...
@@ -11,8 +10,8 @@
 except ImportError:
     from urllib import parse as urlparse
 
-from netrc import netrc
 
+import requests
 from requests.exceptions import RequestException
 
 from smartfile.errors import APIError
@@ -35,8 +34,6 @@ def clean_tokens(*args):
     for i, arg in enumerate(args):
         if len(arg) < 30:
             raise ValueError("too short")
-        if not isinstance(arg, six.text_type):
-            arg = six.u(arg)
         args[i] = arg
     return args
 
@@ -177,8 +174,8 @@ def _do_request(self, *args, **kwargs):
     class OAuthToken(object):
         "Internal representation of an OAuth (token, secret) tuple."
         def __init__(self, token=None, secret=None):
-            self.token = token and six.u(token)
-            self.secret = secret and six.u(secret)
+            self.token = token
+            self.secret = secret
 
         def __iter__(self):
             yield self.token
@@ -229,8 +226,6 @@ def _do_request(self, *args, **kwargs):
 
         def get_request_token(self, callback=None):
             "The first step of the OAuth workflow."
-            if callback:
-                callback = six.u(callback)
             oauth = OAuth1(self._client.token,
                            client_secret=self._client.secret,
                            callback_uri=callback,
@@ -255,8 +250,6 @@ def get_authorization_url(self, request=None):
         def get_access_token(self, request=None, verifier=None):
             """The final step of the OAuth workflow. After this the client can make
             API calls."""
-            if verifier:
-                verifier = six.u(verifier)
             if request is None:
                 if not self.__request.is_valid():
                     raise APIError('You must obtain a request token to request '
@@ -267,7 +260,7 @@ def get_access_token(self, request=None, verifier=None):
                            client_secret=self._client.secret,
                            resource_owner_key=request.token,
                            resource_owner_secret=request.secret,
-                           verifier=six.u(verifier),
+                           verifier=verifier,
                            signature_method=SIGNATURE_PLAINTEXT)
             r = requests.post(urlparse.urljoin(self.url, 'oauth/access_token/'), auth=oauth)
             credentials = urlparse.parse_qs(r.text)

From 8ec2255ac96911b60790199d662f8f8583c7c617 Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@iupui.edu>
Date: Mon, 23 Feb 2015 17:44:37 -0500
Subject: [PATCH 25/90] Updated tests.

---
 tests.py | 117 +++++--------------------------------------------------
 1 file changed, 10 insertions(+), 107 deletions(-)

diff --git a/tests.py b/tests.py
index 2feedbe..dc64db8 100644
--- a/tests.py
+++ b/tests.py
@@ -1,22 +1,13 @@
-import os
-import six
 import cgi
 import json
-import zlib
-import random
-import base64
-import hashlib
+import os
+import tempfile
+import threading
+import unittest
 try:
     import urlparse
 except ImportError:
     import urllib.parse as urlparse
-import unittest
-import tempfile
-import threading
-
-from collections import deque
-
-from six import StringIO
 
 try:
     from BaseHTTPServer import HTTPServer
@@ -27,7 +18,6 @@
 
 from smartfile import BasicClient
 from smartfile import OAuthClient
-from smartfile.sync import SyncClient
 from smartfile.errors import APIError
 from smartfile.errors import RequestError
 
@@ -39,16 +29,6 @@
 ACCESS_TOKEN = 'hIlkipZNmwIJ28HQtQRcbGuXBePQp5'
 ACCESS_SECRET = 'Scen1dwmVtWhjLpJfnilrfdc5OZWCJ'
 
-SYNC_FILE_A = six.b("""This is a test file.
-It will be used with the sync client.""")
-SYNC_FILE_B = six.b("""It will be used with the sync client.
-This is a test file.""")
-
-# b64encode(librsync.signature(SYNC_FILE_A)):
-SYNC_SIGNATURE = base64.b64decode(six.b('cnMBNgAACAAAAAAIFvwbJw0ItorhbKRo'))
-# b64encode(librsync.delta(SYNC_FILE_B, signature)):
-SYNC_DELTA = base64.b64decode(six.b('cnMCNkE6SXQgd2lsbCBiZSB1c2VkIHdpdGggdGhlIHN5bmMgY2xpZW50LgpUaGlzIGlzIGEgdGVzdCBmaWxlLgA='))
-
 
 class TestHTTPRequestHandler(BaseHTTPRequestHandler):
     """
@@ -77,7 +57,7 @@ def respond(self, request):
         self.send_response(200)
         self.send_header("Content-type", "text/plain")
         self.end_headers()
-        self.wfile.write(six.b("Hello World!"))
+        self.wfile.write(b"Hello World!")
 
     def parse_and_record(self, method):
         urlp = urlparse.urlparse(self.path)
@@ -248,7 +228,7 @@ class DownloadTestCase(object):
     def test_file_response(self):
         r = self.client.get('/user')
         self.assertTrue(hasattr(r, 'read'), 'File-like object not returned.')
-        self.assertEqual(r.read(), six.b('Hello World!'))
+        self.assertEqual(r.read(), b'Hello World!')
 
 
 class UploadTestCase(object):
@@ -315,8 +295,8 @@ def test_netrc(self):
                     address, port = address
                 else:
                     port = self.server.server_port
-                netrc = six.b("machine 127.0.0.1:%s\n  login %s\n  password %s" % (
-                        port, API_KEY, API_PASSWORD))
+                netrc = b"machine 127.0.0.1:%s\n  login %s\n  password %s" % (
+                        port, API_KEY, API_PASSWORD)
                 os.write(fd, netrc)
             finally:
                 os.close(fd)
@@ -346,7 +326,7 @@ def respond(self, request):
         self.send_response(503)
         self.send_header("X-Throttle", "throttled; next=0.01 sec")
         self.end_headers()
-        self.wfile.write(six.b("Request Throttled!"))
+        self.wfile.write(b"Request Throttled!")
 
 
 class ThrottleTestCase(object):
@@ -370,7 +350,7 @@ def respond(self, request):
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
-        self.wfile.write(six.b(json.dumps({ 'foo': 'bar' })))
+        self.wfile.write(b"%s" % (json.dumps({ 'foo': 'bar' }), ))
 
 
 class JSONTestCase(object):
@@ -390,83 +370,6 @@ class OAuthJSONTestCase(JSONTestCase, OAuthTestCase):
     pass
 
 
-class SyncRequestHandler(TestHTTPRequestHandler):
-    def respond(self, request):
-        if '/signature/' in request.path:
-            return self.respond_signature()
-        elif '/delta/' in request.path:
-            return self.respond_delta()
-        elif '/patch/' in request.path:
-            return self.respond_patch()
-        else:
-            return super(SyncRequestHandler, self).respond()
-
-    def respond_signature(self):
-        self.send_response(200)
-        self.send_header("Content-Type", "application/librsync-signature")
-        self.end_headers()
-        self.wfile.write(SYNC_SIGNATURE)
-
-    def respond_delta(self):
-        self.send_response(200)
-        self.send_header("Content-Type", "application/librsync-delta")
-        self.end_headers()
-        self.wfile.write(SYNC_DELTA)
-
-    def respond_patch(self):
-        self.send_response(200)
-        self.send_header("Content-Type", "application/json")
-        self.end_headers()
-        self.wfile.write(six.b(json.dumps({'foo': 'bar'})))
-
-
-class SyncTestCase(object):
-    "Test delta transfer via sync API."
-    handler = SyncRequestHandler
-
-    def getClient(self, **kwargs):
-        "Override to return a sync client that uses the underlying client."
-        client = super(SyncTestCase, self).getClient(**kwargs)
-        return SyncClient(client)
-
-    def test_upload(self):
-        "Ensure we can upload via sync API."
-        fd, t = tempfile.mkstemp()
-        os.write(fd, SYNC_FILE_B)
-        os.close(fd)
-        try:
-            self.client.upload(t, '/unittest/sync')
-        finally:
-            os.unlink(t)
-        self.assertRequestCount(2)
-        self.assertPath('/api/%s/path/sync/signature/unittest/sync/' % self.client.version, request=0)
-        self.assertPath('/api/%s/path/sync/patch/unittest/sync/' % self.client.version, request=1)
-        self.assertData('delta', SYNC_DELTA, request=1)
-
-    def test_download(self):
-        "Ensure we can download via sync API."
-        fd, t = tempfile.mkstemp()
-        os.write(fd, SYNC_FILE_A)
-        os.close(fd)
-        try:
-            self.client.download(t, '/unittest/sync')
-            # The local file contents should have been changed.
-            self.assertEqual(SYNC_FILE_B, open(t, 'rb').read())
-        finally:
-            os.unlink(t)
-        self.assertRequestCount(1)
-        self.assertPath('/api/%s/path/sync/delta/unittest/sync/' % self.client.version)
-        self.assertData('signature', SYNC_SIGNATURE)
-
-
-class BasicSyncTestCase(SyncTestCase, BasicTestCase):
-    pass
-
-
-class OAuthSyncTestCase(SyncTestCase, OAuthTestCase):
-    pass
-
-
 # TODO: Test with missing oauthlib...
 # Must invoke an ImportError when smartfile tries to import it. Then the test
 # case should verify that the correct exception (NotImplementedError) is raised

From 9589ad0843850881afe7efb788d4602c2401ed77 Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@iupui.edu>
Date: Mon, 23 Feb 2015 17:48:56 -0500
Subject: [PATCH 26/90] Removed librsync support.

---
 smartfile/errors.py |   7 +--
 smartfile/sync.py   | 121 --------------------------------------------
 2 files changed, 2 insertions(+), 126 deletions(-)
 delete mode 100644 smartfile/sync.py

diff --git a/smartfile/errors.py b/smartfile/errors.py
index 758844a..4b64c13 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -1,6 +1,3 @@
-import six
-
-
 class APIError(Exception):
     "SmartFile API base Exception."
     pass
@@ -26,9 +23,9 @@ def __init__(self, response, *args, **kwargs):
             json = response.json()
         except ValueError:
             if self.status_code == 404:
-                self.detail = six.u('Invalid URL, check your API path')
+                self.detail = 'Invalid URL, check your API path'
             else:
-                self.detail = six.u('Server error; check response for errors')
+                self.detail = 'Server error; check response for errors'
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']
diff --git a/smartfile/sync.py b/smartfile/sync.py
deleted file mode 100644
index 861711a..0000000
--- a/smartfile/sync.py
+++ /dev/null
@@ -1,121 +0,0 @@
-import os
-import errno
-import tempfile
-
-try:
-    import librsync
-except ImportError:
-    raise ImportError('python-librsync is required for sync capabilities. '
-                      'Install it using `pip install python-librsync`.')
-
-
-class BaseFile(object):
-    """
-    Base class for files being synchronized.
-    """
-    def __init__(self, path):
-        self.path = path
-
-
-class LocalFile(BaseFile):
-    """
-    Represents a local file that is being synchronized. Uses librsync to
-    perform the steps of the rsync algorithm.
-    """
-    def signature(self, block_size=None):
-        "Calculates signature for local file."
-        kwargs = {}
-        if block_size:
-            kwargs['block_size'] = block_size
-        return librsync.signature(open(self.path, 'rb'), **kwargs)
-
-    def delta(self, signature):
-        "Generates delta for local file using remote signature."
-        return librsync.delta(open(self.path, 'rb'), signature)
-
-    def patch(self, delta):
-        "Applies remote delta to local file."
-        # Create a temp file in which to store our synced copy. We will handle
-        # deleting it manually, since we may move it instead.
-        with (tempfile.NamedTemporaryFile(prefix='.sync',
-              suffix=os.path.basename(self.path),
-              dir=os.path.dirname(self.path), delete=False)) as output:
-            try:
-                # Open the local file, data may be read from it.
-                with open(self.path, 'rb') as reference:
-                    # Patch the local file into our temporary file.
-                    r = librsync.patch(reference, delta, output)
-                    os.rename(output.name, self.path)
-                    return r
-            finally:
-                try:
-                    os.remove(output.name)
-                except OSError as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-
-
-class RemoteFile(BaseFile):
-    """
-    Represents a remote file that is being synchronized. Makes API calls to
-    perform the steps of the rsync algorithm.
-    """
-    def __init__(self, path, api):
-        super(RemoteFile, self).__init__(path)
-        self.api = api
-
-    def signature(self, block_size=None):
-        "Requests a signature for remote file via API."
-        kwargs = {}
-        if block_size:
-            kwargs['block_size'] = block_size
-        return self.api.get('path/sync/signature', self.path, **kwargs)
-
-    def delta(self, signature):
-        "Generates delta for remote file via API using local file's signature."
-        return self.api.post('path/sync/delta', self.path, signature=signature)
-
-    def patch(self, delta):
-        "Applies delta for local file to remote file via API."
-        return self.api.post('path/sync/patch', self.path, delta=delta)
-
-
-class SyncClient(object):
-    """
-    Synchronizes remote and local files.
-    """
-    def __init__(self, api, block_size=None):
-        """
-        Synchronizes files with SmartFile using the sync API.
-        """
-        self.api = api
-        self.block_size = block_size
-
-    @property
-    def version(self):
-        return self.api.version
-
-    def sync(self, src, dst):
-        """
-        Performs synchronization from source to destination. Performs the three
-        steps:
-
-        1. Calculate signature of destination.
-        2. Generate delta from source.
-        3. Apply delta to destination.
-        """
-        return dst.patch(src.delta(dst.signature(block_size=self.block_size)))
-
-    def upload(self, local, remote):
-        """
-        Performs synchronization from a local file to a remote file. The local
-        path is the source and remote path is the destination.
-        """
-        self.sync(LocalFile(local), RemoteFile(remote, self.api))
-
-    def download(self, local, remote):
-        """
-        Performs synchronization from a remote file to a local file. The
-        remote path is the source and the local path is the destination.
-        """
-        self.sync(RemoteFile(remote, self.api), LocalFile(local))

From eb519844afea8cb5bcef63ab4e7d5451141e6fc1 Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@iupui.edu>
Date: Mon, 23 Feb 2015 18:17:22 -0500
Subject: [PATCH 27/90] Revert changes

---
 .travis.yml         |   2 -
 smartfile/errors.py |   7 ++-
 smartfile/sync.py   | 121 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 126 insertions(+), 4 deletions(-)
 create mode 100644 smartfile/sync.py

diff --git a/.travis.yml b/.travis.yml
index 09b9332..b55be14 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,5 +16,3 @@ script:
   - make test
 after_success:
   - coveralls
-notifications:
-  slack: smartfile:tbDIPzVJIPBpSz29kQw6b8RQ
diff --git a/smartfile/errors.py b/smartfile/errors.py
index 4b64c13..758844a 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -1,3 +1,6 @@
+import six
+
+
 class APIError(Exception):
     "SmartFile API base Exception."
     pass
@@ -23,9 +26,9 @@ def __init__(self, response, *args, **kwargs):
             json = response.json()
         except ValueError:
             if self.status_code == 404:
-                self.detail = 'Invalid URL, check your API path'
+                self.detail = six.u('Invalid URL, check your API path')
             else:
-                self.detail = 'Server error; check response for errors'
+                self.detail = six.u('Server error; check response for errors')
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']
diff --git a/smartfile/sync.py b/smartfile/sync.py
new file mode 100644
index 0000000..861711a
--- /dev/null
+++ b/smartfile/sync.py
@@ -0,0 +1,121 @@
+import os
+import errno
+import tempfile
+
+try:
+    import librsync
+except ImportError:
+    raise ImportError('python-librsync is required for sync capabilities. '
+                      'Install it using `pip install python-librsync`.')
+
+
+class BaseFile(object):
+    """
+    Base class for files being synchronized.
+    """
+    def __init__(self, path):
+        self.path = path
+
+
+class LocalFile(BaseFile):
+    """
+    Represents a local file that is being synchronized. Uses librsync to
+    perform the steps of the rsync algorithm.
+    """
+    def signature(self, block_size=None):
+        "Calculates signature for local file."
+        kwargs = {}
+        if block_size:
+            kwargs['block_size'] = block_size
+        return librsync.signature(open(self.path, 'rb'), **kwargs)
+
+    def delta(self, signature):
+        "Generates delta for local file using remote signature."
+        return librsync.delta(open(self.path, 'rb'), signature)
+
+    def patch(self, delta):
+        "Applies remote delta to local file."
+        # Create a temp file in which to store our synced copy. We will handle
+        # deleting it manually, since we may move it instead.
+        with (tempfile.NamedTemporaryFile(prefix='.sync',
+              suffix=os.path.basename(self.path),
+              dir=os.path.dirname(self.path), delete=False)) as output:
+            try:
+                # Open the local file, data may be read from it.
+                with open(self.path, 'rb') as reference:
+                    # Patch the local file into our temporary file.
+                    r = librsync.patch(reference, delta, output)
+                    os.rename(output.name, self.path)
+                    return r
+            finally:
+                try:
+                    os.remove(output.name)
+                except OSError as e:
+                    if e.errno != errno.ENOENT:
+                        raise
+
+
+class RemoteFile(BaseFile):
+    """
+    Represents a remote file that is being synchronized. Makes API calls to
+    perform the steps of the rsync algorithm.
+    """
+    def __init__(self, path, api):
+        super(RemoteFile, self).__init__(path)
+        self.api = api
+
+    def signature(self, block_size=None):
+        "Requests a signature for remote file via API."
+        kwargs = {}
+        if block_size:
+            kwargs['block_size'] = block_size
+        return self.api.get('path/sync/signature', self.path, **kwargs)
+
+    def delta(self, signature):
+        "Generates delta for remote file via API using local file's signature."
+        return self.api.post('path/sync/delta', self.path, signature=signature)
+
+    def patch(self, delta):
+        "Applies delta for local file to remote file via API."
+        return self.api.post('path/sync/patch', self.path, delta=delta)
+
+
+class SyncClient(object):
+    """
+    Synchronizes remote and local files.
+    """
+    def __init__(self, api, block_size=None):
+        """
+        Synchronizes files with SmartFile using the sync API.
+        """
+        self.api = api
+        self.block_size = block_size
+
+    @property
+    def version(self):
+        return self.api.version
+
+    def sync(self, src, dst):
+        """
+        Performs synchronization from source to destination. Performs the three
+        steps:
+
+        1. Calculate signature of destination.
+        2. Generate delta from source.
+        3. Apply delta to destination.
+        """
+        return dst.patch(src.delta(dst.signature(block_size=self.block_size)))
+
+    def upload(self, local, remote):
+        """
+        Performs synchronization from a local file to a remote file. The local
+        path is the source and remote path is the destination.
+        """
+        self.sync(LocalFile(local), RemoteFile(remote, self.api))
+
+    def download(self, local, remote):
+        """
+        Performs synchronization from a remote file to a local file. The
+        remote path is the source and the local path is the destination.
+        """
+        self.sync(RemoteFile(remote, self.api), LocalFile(local))

From 04ff2ce406d98f3a299ab956c476ba27feae8956 Mon Sep 17 00:00:00 2001
From: Travis Cunningham <travcunn@umail.iu.edu>
Date: Tue, 24 Feb 2015 18:01:12 -0500
Subject: [PATCH 28/90] Update requirements.txt

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index a1563cd..3faf34c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+six
 oauthlib
 requests
 requests_oauthlib

From 00218c79700cfba6e2037dc3a7513ac6ca0ec868 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Thu, 19 May 2016 16:47:04 -0400
Subject: [PATCH 29/90] add upload method

---
 .gitignore | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 91 insertions(+)
 create mode 100644 .gitignore

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f1f7d42
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,91 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+main.py
\ No newline at end of file

From 667e7ec02bd65e8edc0985c86996f21b0b3bbc14 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Thu, 19 May 2016 16:50:43 -0400
Subject: [PATCH 30/90] actually added upload method this time

---
 smartfile/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index c929179..3048480 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -126,6 +126,9 @@ def post(self, endpoint, id=None, **kwargs):
 
     def delete(self, endpoint, id=None, **kwargs):
         return self._request('delete', endpoint, id=id, data=kwargs)
+        
+    def upload(self, usrfile):
+        return self.post('/path/data/', file=usrfile)    
 
 
 class BasicClient(Client):

From 72d6e3a0c355bb3ee5a29835f9d6b9a38c293348 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 10:23:10 -0400
Subject: [PATCH 31/90] added download method

---
 smartfile/.vscode/launch.json | 63 +++++++++++++++++++++++++++++++++++
 smartfile/__init__.py         |  7 +++-
 2 files changed, 69 insertions(+), 1 deletion(-)
 create mode 100644 smartfile/.vscode/launch.json

diff --git a/smartfile/.vscode/launch.json b/smartfile/.vscode/launch.json
new file mode 100644
index 0000000..f93b136
--- /dev/null
+++ b/smartfile/.vscode/launch.json
@@ -0,0 +1,63 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python",
+            "type": "python",
+            "request": "launch",
+            "stopOnEntry": true,
+            "program": "${file}",
+            "debugOptions": [
+                "WaitOnAbnormalExit",
+                "WaitOnNormalExit",
+                "RedirectOutput"
+            ]
+        },
+        {
+            "name": "Python Console App",
+            "type": "python",
+            "request": "launch",
+            "stopOnEntry": true,
+            "program": "${file}",
+            "externalConsole": true,
+            "debugOptions": [
+                "WaitOnAbnormalExit",
+                "WaitOnNormalExit"
+            ]
+        },
+        {
+            "name": "Django",
+            "type": "python",
+            "request": "launch",
+            "stopOnEntry": true,
+            "program": "${workspaceRoot}/manage.py",
+            "args": [
+                "runserver",
+                "--noreload"
+            ],
+            "debugOptions": [
+                "WaitOnAbnormalExit",
+                "WaitOnNormalExit",
+                "RedirectOutput",
+                "DjangoDebugging"
+            ]
+        },
+        {
+            "name": "Watson",
+            "type": "python",
+            "request": "launch",
+            "stopOnEntry": true,
+            "program": "${workspaceRoot}/console.py",
+            "args": [
+                "dev",
+                "runserver",
+                "--noreload=True"
+            ],
+            "debugOptions": [
+                "WaitOnAbnormalExit",
+                "WaitOnNormalExit",
+                "RedirectOutput"
+            ]
+        }
+    ]
+}
\ No newline at end of file
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 3048480..fa9d82d 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -128,7 +128,12 @@ def delete(self, endpoint, id=None, **kwargs):
         return self._request('delete', endpoint, id=id, data=kwargs)
         
     def upload(self, usrfile):
-        return self.post('/path/data/', file=usrfile)    
+        return self.post('/path/data/', file=usrfile)  
+        
+    def download(self, downloadfile):
+        return self.get('/path/data/', downloadfile)      
+        
+      
 
 
 class BasicClient(Client):

From 3bdaabdd49d5199c3247d399bedfc7df040ce605 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 10:34:07 -0400
Subject: [PATCH 32/90] update README

---
 README.rst | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index 61c2509..70c947f 100644
--- a/README.rst
+++ b/README.rst
@@ -208,9 +208,9 @@ To upload a file, pass either a file-like object or a tuple of
     >>> data = StringIO('StringIO instance has no .name attribute!')
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> api.post('/path/data/', file=('foobar.png', data))
-    >>> # Or use a file-like object with a name attribute
-    >>> api.post('/path/data/', file=file('foobar.png', 'rb'))
+    >>> f = ('foobar.png', data)
+    >>> api.upload(f)
+
 
 Downloading is automatic, if the ``'Content-Type'`` header indicates
 content other than the expected JSON return value, then a file-like object is
@@ -221,10 +221,11 @@ returned.
     >>> import shutil
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> f = api.get('/path/data/', 'foobar.png')
+    >>> f = api.download('foobar.png')
     >>> with file('foobar.png', 'wb') as o:
     >>>     shutil.copyfileobj(f, o)
 
+
 Tasks
 -----
 

From 1a6072a696b9112c6e0f2dbaf2c6346bec56b789 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 11:51:21 -0400
Subject: [PATCH 33/90] added move method

---
 smartfile/__init__.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index fa9d82d..81ba1ea 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -131,7 +131,23 @@ def upload(self, usrfile):
         return self.post('/path/data/', file=usrfile)  
         
     def download(self, downloadfile):
-        return self.get('/path/data/', downloadfile)      
+        return self.get('/path/data/', downloadfile)
+    
+    def move(self, sourcefile, destination):
+        # check destination folder for / at end
+        if destination[-1:] != "/":
+            raise Exception("Destination folder requires a / at end")
+        # check destination folder for / at begining
+        if destination[:-1] != "/":
+            destination = "/" + destination
+        try:
+            t = self.post('/path/oper/move/', src=sourcefile, dst=destination)
+        except KeyError:
+            raise Exception("Destination directory does not exist")
+        while True:
+            s = self.get('/task', t['uuid'])
+            if s['result']['status'] == 'SUCCESS':
+                break
         
       
 

From 5d55f96a40107f3d96825a86d70ec83efef96a60 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 11:53:26 -0400
Subject: [PATCH 34/90] updated README for move

---
 README.rst | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index 70c947f..15c7c66 100644
--- a/README.rst
+++ b/README.rst
@@ -237,11 +237,7 @@ to poll the status of the task.
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> t = api.post('/path/oper/move/', src='/foobar.png', dst='/images/foobar.png')
-    >>> while True:
-    >>>     s = api.get('/task', t['uuid'])
-    >>>     if s['status'] == 'SUCCESS':
-    >>>         break
+    >>> api.move('/foobar.png', 'Folder/')
 
 
 .. _SmartFile: http://www.smartfile.com/

From 449ae55413b75ed7cbadd594efcfdf0f06993b6d Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 11:55:11 -0400
Subject: [PATCH 35/90] updated README again

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 15c7c66..16951b6 100644
--- a/README.rst
+++ b/README.rst
@@ -237,7 +237,7 @@ to poll the status of the task.
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> api.move('/foobar.png', 'Folder/')
+    >>> api.move('/foobar.png', '/Folder/')
 
 
 .. _SmartFile: http://www.smartfile.com/

From fd9afb47cc2d93b55d68c71759787ae3e3228c69 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 12:02:16 -0400
Subject: [PATCH 36/90] update README. See #{3685}

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 16951b6..8a04250 100644
--- a/README.rst
+++ b/README.rst
@@ -241,4 +241,4 @@ to poll the status of the task.
 
 
 .. _SmartFile: http://www.smartfile.com/
-.. _Read more: http://www.smartfile.com/open-source.html
+.. _Read more: http://www.smartfile.com/open-source.html
\ No newline at end of file

From 66ed7d2f4da83748fdec13c404453b4c1213734e Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 12:04:31 -0400
Subject: [PATCH 37/90] update README. See #3685

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 8a04250..16951b6 100644
--- a/README.rst
+++ b/README.rst
@@ -241,4 +241,4 @@ to poll the status of the task.
 
 
 .. _SmartFile: http://www.smartfile.com/
-.. _Read more: http://www.smartfile.com/open-source.html
\ No newline at end of file
+.. _Read more: http://www.smartfile.com/open-source.html

From 3ab0b1734a674ea8ea270e574a3def8f875c7413 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 17:49:55 -0400
Subject: [PATCH 38/90] exception added for upload

---
 smartfile/__init__.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 81ba1ea..a33b24f 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -128,7 +128,14 @@ def delete(self, endpoint, id=None, **kwargs):
         return self._request('delete', endpoint, id=id, data=kwargs)
         
     def upload(self, usrfile):
-        return self.post('/path/data/', file=usrfile)  
+        # needed to split the tuple
+        newtuple = usrfile
+        # splits tuple to allow ability to remove "/" at the end, if present 
+        ourfile, newdata = newtuple
+        if ourfile[-1:] == "/":
+             ourfile = ourfile[:-1]
+        newtuple = (ourfile, newdata)
+        return self.post('/path/data/', file=newtuple)  
         
     def download(self, downloadfile):
         return self.get('/path/data/', downloadfile)

From daa7f30ea611b1fcedc2ecccc81b03f7abf47940 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 20 May 2016 17:51:49 -0400
Subject: [PATCH 39/90] update exception for upload

---
 smartfile/__init__.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index a33b24f..7446431 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -133,9 +133,8 @@ def upload(self, usrfile):
         # splits tuple to allow ability to remove "/" at the end, if present 
         ourfile, newdata = newtuple
         if ourfile[-1:] == "/":
-             ourfile = ourfile[:-1]
-        newtuple = (ourfile, newdata)
-        return self.post('/path/data/', file=newtuple)  
+             raise Exception("Can only upload files")
+        return self.post('/path/data/', file=usrfile)  
         
     def download(self, downloadfile):
         return self.get('/path/data/', downloadfile)
@@ -155,8 +154,7 @@ def move(self, sourcefile, destination):
             s = self.get('/task', t['uuid'])
             if s['result']['status'] == 'SUCCESS':
                 break
-        
-      
+           
 
 
 class BasicClient(Client):

From a74a5aa7b06e7bc043d354b501a147238f4478ed Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Mon, 23 May 2016 16:58:36 -0400
Subject: [PATCH 40/90] added testing for updated operations

---
 test/test_smartfile.py | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 test/test_smartfile.py

diff --git a/test/test_smartfile.py b/test/test_smartfile.py
new file mode 100644
index 0000000..7fd93e6
--- /dev/null
+++ b/test/test_smartfile.py
@@ -0,0 +1,32 @@
+import os
+import unittest
+
+from smartfile import BasicClient
+
+API_KEY = os.environ.get("API_KEY")
+API_PASSWORD = os.environ.get("API_PASSWORD")
+
+if API_KEY is None:
+    raise RuntimeError("API_KEY is required")
+    
+if API_PASSWORD is None:
+    raise RuntimeError("API_PASSWORD is required")
+    
+
+class CustomOperationsTestCase(unittest.TestCase): 
+    
+    def setUp(self):
+        self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
+        self.api = BasicClient(API_KEY, API_PASSWORD)
+        self.txtfile = self.current_dir + "resources/myfile.txt"
+  
+    def test_upload_download(self):
+        data = open(self.txtfile, "rb")
+        newfile = ('myfile.txt', data)
+        self.api.upload(newfile)
+
+        f = self.api.download("myfile.txt")
+        self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
+    
+    def test_move(self):
+        self.api.move('README.rst', '/newFolder/')
\ No newline at end of file

From 0ff9860a4bab4146b16a0b4e65797c8773509353 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 24 May 2016 12:05:19 -0400
Subject: [PATCH 41/90] update .gitignore for test files

---
 .gitignore | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index f1f7d42..d9b067b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -88,4 +88,6 @@ ENV/
 # Rope project settings
 .ropeproject
 
-main.py
\ No newline at end of file
+main.py
+
+/test/resources
\ No newline at end of file

From 6877a5bdede807b482081e1e7f77d6fb828a1b1b Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 24 May 2016 12:06:12 -0400
Subject: [PATCH 42/90] updated tests

---
 test/test_smartfile.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 7fd93e6..13f3e51 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -19,12 +19,22 @@ def setUp(self):
         self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
         self.api = BasicClient(API_KEY, API_PASSWORD)
         self.txtfile = self.current_dir + "resources/myfile.txt"
-  
+        self.uploaddata = None
+    
+    def test_delete(self):
+        
+    
+    def get_data(self):
+        self.uploaddata = self.api.get("/path/info/myfile.txt")
+        return self.uploaddata
+    
     def test_upload_download(self):
         data = open(self.txtfile, "rb")
         newfile = ('myfile.txt', data)
         self.api.upload(newfile)
-
+        self.assertEquals(self.get_data()['size'], os.path.getsize(self.txtfile))
+        
+        
         f = self.api.download("myfile.txt")
         self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
     

From d8a99114e131038b57ae98832788417cbfc1c38d Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 24 May 2016 12:06:47 -0400
Subject: [PATCH 43/90] updated delete method

---
 smartfile/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 7446431..b30f0f3 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -124,8 +124,8 @@ def put(self, endpoint, id=None, **kwargs):
     def post(self, endpoint, id=None, **kwargs):
         return self._request('post', endpoint, id=id, data=kwargs)
 
-    def delete(self, endpoint, id=None, **kwargs):
-        return self._request('delete', endpoint, id=id, data=kwargs)
+    def delete(self, deletefile):
+        return self.post('/path/oper/remove', path=deletefile)
         
     def upload(self, usrfile):
         # needed to split the tuple

From eb4faac86e3c8746eb6b372e07b3819892a44fa8 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 24 May 2016 14:39:20 -0400
Subject: [PATCH 44/90] update delete function to raise exception

---
 smartfile/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index b30f0f3..42350a6 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -125,7 +125,10 @@ def post(self, endpoint, id=None, **kwargs):
         return self._request('post', endpoint, id=id, data=kwargs)
 
     def delete(self, deletefile):
-        return self.post('/path/oper/remove', path=deletefile)
+        try:
+            return self.post('/path/oper/remove', path=deletefile)
+        except KeyError:
+            raise Exception("Destination file does not exist")
         
     def upload(self, usrfile):
         # needed to split the tuple

From cb42d325f2741c58f5e4cbb186da948d2bcb45df Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 24 May 2016 14:42:05 -0400
Subject: [PATCH 45/90] added and updated testing

---
 test/test_smartfile.py | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 13f3e51..728ab72 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -20,23 +20,37 @@ def setUp(self):
         self.api = BasicClient(API_KEY, API_PASSWORD)
         self.txtfile = self.current_dir + "resources/myfile.txt"
         self.uploaddata = None
-    
-    def test_delete(self):
-        
+
+    def clean_up(self):
+        self.api.move("/newFolder/README.rst", "../")
     
     def get_data(self):
         self.uploaddata = self.api.get("/path/info/myfile.txt")
         return self.uploaddata
-    
-    def test_upload_download(self):
+        
+    def upload(self):
         data = open(self.txtfile, "rb")
         newfile = ('myfile.txt', data)
         self.api.upload(newfile)
         self.assertEquals(self.get_data()['size'], os.path.getsize(self.txtfile))
         
-        
+    def download(self): 
         f = self.api.download("myfile.txt")
         self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
     
-    def test_move(self):
-        self.api.move('README.rst', '/newFolder/')
\ No newline at end of file
+    def move(self):
+        self.api.move('README.rst', '/newFolder/')
+        
+    def delete(self):
+        data = open(self.txtfile, "rb")
+        newfile = ('myfile.txt', data)
+        self.api.upload(newfile)
+        self.api.delete("myfile.txt")
+        self.assertRaises(Exception, BasicClient.delete) 
+        
+    def test_upload_download_move_delete_clean_up(self):
+        self.upload()
+        self.download()
+        self.move()
+        self.delete()
+        self.clean_up()
\ No newline at end of file

From 55cd43819a06feab0a08d04b01fe3d172d4ddf7c Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 24 May 2016 15:36:10 -0400
Subject: [PATCH 46/90] Removed dependencies in testing

---
 test/test_smartfile.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 728ab72..1cb885a 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -20,9 +20,6 @@ def setUp(self):
         self.api = BasicClient(API_KEY, API_PASSWORD)
         self.txtfile = self.current_dir + "resources/myfile.txt"
         self.uploaddata = None
-
-    def clean_up(self):
-        self.api.move("/newFolder/README.rst", "../")
     
     def get_data(self):
         self.uploaddata = self.api.get("/path/info/myfile.txt")
@@ -39,18 +36,14 @@ def download(self):
         self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
     
     def move(self):
-        self.api.move('README.rst', '/newFolder/')
+        self.api.move('myfile.txt', '/newFolder/')
         
     def delete(self):
-        data = open(self.txtfile, "rb")
-        newfile = ('myfile.txt', data)
-        self.api.upload(newfile)
-        self.api.delete("myfile.txt")
+        self.api.delete("/newFolder/myfile.txt")
         self.assertRaises(Exception, BasicClient.delete) 
         
     def test_upload_download_move_delete_clean_up(self):
         self.upload()
         self.download()
         self.move()
-        self.delete()
-        self.clean_up()
\ No newline at end of file
+        self.delete()
\ No newline at end of file

From d9a29f89924b52399a2d6d16f1016d0376ee8f21 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 24 May 2016 15:39:43 -0400
Subject: [PATCH 47/90] Update README to reflect changes

---
 README.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/README.rst b/README.rst
index 16951b6..0d5690f 100644
--- a/README.rst
+++ b/README.rst
@@ -233,12 +233,22 @@ Operations are long-running jobs that are not executed within the time frame
 of an API call. For such operations, a task is created, and the API can be used
 to poll the status of the task.
 
+Move files
+
 .. code:: python
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
     >>> api.move('/foobar.png', '/Folder/')
+    
+    
+Delete files
+
+.. code:: python
 
+    >>> from smartfile import BasicClient
+    >>> api = BasicClient()
+    >>> api.delete('/foobar.png')
 
 .. _SmartFile: http://www.smartfile.com/
 .. _Read more: http://www.smartfile.com/open-source.html

From 8f4f74f4b829898872765c5cee8ca6fcd6c96e70 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 25 May 2016 15:41:59 -0400
Subject: [PATCH 48/90] addressed all comments in PR. See #3685

---
 README.rst             | 14 +++----
 smartfile/__init__.py  | 89 ++++++++++++++++++++++--------------------
 smartfile/errors.py    |  2 +-
 test/test_smartfile.py | 31 ++++++++-------
 4 files changed, 69 insertions(+), 67 deletions(-)

diff --git a/README.rst b/README.rst
index 0d5690f..e37ee1c 100644
--- a/README.rst
+++ b/README.rst
@@ -142,7 +142,7 @@ Authentication using OAuth authentication is bit more complicated, as it involve
     >>> from smartfile import OAuthClient
     >>> api = OAuthClient('**********', '**********')
     >>> # Be sure to only call each method once for each OAuth login
-    >>> 
+    >>>
     >>> # This is the first step with the client, which should be left alone
     >>> api.get_request_token()
     >>> # Redirect users to the following URL:
@@ -199,16 +199,14 @@ File transfers
 
 Uploading and downloading files is supported.
 
-To upload a file, pass either a file-like object or a tuple of
-``(filename, file-like)`` as a kwarg.
+To upload a file, pass a tuple of (filename, fileobject). For example:
 
 .. code:: python
 
-    >>> from StringIO import StringIO
-    >>> data = StringIO('StringIO instance has no .name attribute!')
+    >>> data = file('Song.mp3', 'rb')
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> f = ('foobar.png', data)
+    >>> f = ('Song.mp3', data)
     >>> api.upload(f)
 
 
@@ -240,8 +238,8 @@ Move files
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
     >>> api.move('/foobar.png', '/Folder/')
-    
-    
+
+
 Delete files
 
 .. code:: python
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 42350a6..98d8a33 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -54,7 +54,8 @@ def _do_request(self, request, url, **kwargs):
         else:
             if response.status_code >= 400:
                 raise ResponseError(response)
-        # Try to return the response in the most useful fashion given it's type.
+        # Try to return the response in the most useful fashion given it's
+        # type.
         if response.headers.get('content-type') == 'application/json':
             try:
                 # Try to decode as JSON
@@ -94,18 +95,21 @@ def _request(self, method, endpoint, id=None, **kwargs):
             path = path.replace('//', '/')
         url = self.url + path
         # Add our user agent.
-        kwargs.setdefault('headers', {}).setdefault('User-Agent', HTTP_USER_AGENT)
+        kwargs.setdefault('headers', {}).setdefault('User-Agent',
+                                                    HTTP_USER_AGENT)
         # Now try the request, if we get throttled, sleep and try again.
         trys, retrys = 0, 3
         while True:
             if trys == retrys:
-                raise RequestError('Could not complete request after %s trys.' % trys)
+                raise RequestError('Could not complete request after %s trys.'
+                                   % trys)
             trys += 1
             try:
                 return self._do_request(request, url, **kwargs)
             except ResponseError as e:
                 if self.throttle_wait and e.status_code == 503:
-                    m = THROTTLE_PATTERN.match(e.response.headers.get('x-throttle', ''))
+                    m = THROTTLE_PATTERN.match(
+                        e.response.headers.get('x-throttle', ''))
                     if m:
                         time.sleep(float(m.group(1)))
                         continue
@@ -129,35 +133,29 @@ def delete(self, deletefile):
             return self.post('/path/oper/remove', path=deletefile)
         except KeyError:
             raise Exception("Destination file does not exist")
-        
+
     def upload(self, usrfile):
-        # needed to split the tuple
-        newtuple = usrfile
-        # splits tuple to allow ability to remove "/" at the end, if present 
-        ourfile, newdata = newtuple
-        if ourfile[-1:] == "/":
-             raise Exception("Can only upload files")
-        return self.post('/path/data/', file=usrfile)  
-        
-    def download(self, downloadfile):
-        return self.get('/path/data/', downloadfile)
-    
+        if usrfile[0].endswith('/'):
+            raise ValueError("File name should have no trailing slash")
+        return self.post('/path/data/', file=usrfile)
+
+    def download(self, file_to_be_downloaded):
+        """ file_to_be_downloaded is a file-like object that has already
+        been uploaded, you cannot download folders """
+        return self.get('/path/data/', file_to_be_downloaded)
+
     def move(self, sourcefile, destination):
         # check destination folder for / at end
-        if destination[-1:] != "/":
-            raise Exception("Destination folder requires a / at end")
+        if destination.endswith("/"):
+            destination = destination + "/"
         # check destination folder for / at begining
-        if destination[:-1] != "/":
+        if destination.startswith("/"):
             destination = "/" + destination
-        try:
-            t = self.post('/path/oper/move/', src=sourcefile, dst=destination)
-        except KeyError:
-            raise Exception("Destination directory does not exist")
+        t = self.post('/path/oper/move/', src=sourcefile, dst=destination)
         while True:
             s = self.get('/task', t['uuid'])
             if s['result']['status'] == 'SUCCESS':
                 break
-           
 
 
 class BasicClient(Client):
@@ -225,10 +223,11 @@ def is_valid(self):
                 return False
 
     class OAuthClient(Client):
-        """API client that uses OAuth tokens. Layers a more complex form of
-        authentication useful for 3rd party access on top of the base Client."""
-        def __init__(self, client_token=None, client_secret=None, access_token=None,
-                     access_secret=None, **kwargs):
+        """API client that uses OAuth tokens. Layers a more complex
+        form of authentication useful for 3rd party access on top of
+        the base Client."""
+        def __init__(self, client_token=None, client_secret=None,
+                     access_token=None, access_secret=None, **kwargs):
             if client_token is None:
                 client_token = os.environ.get('SMARTFILE_CLIENT_TOKEN')
             if client_secret is None:
@@ -239,15 +238,15 @@ def __init__(self, client_token=None, client_secret=None, access_token=None,
                 access_secret = os.environ.get('SMARTFILE_ACCESS_SECRET')
             self._client = OAuthToken(client_token, client_secret)
             if not self._client.is_valid():
-                raise APIError('You must provide a client_token and client_secret '
-                               'for OAuth.')
+                raise APIError('You must provide a client_token'
+                               'and client_secret for OAuth.')
             self._access = OAuthToken(access_token, access_secret)
             super(OAuthClient, self).__init__(**kwargs)
 
         def _do_request(self, *args, **kwargs):
             if not self._access.is_valid():
-                raise APIError('You must obtain an access token before making API '
-                               'calls.')
+                raise APIError('You must obtain an access token'
+                               'before making API calls.')
             # Add the OAuth parameters.
             kwargs['auth'] = OAuth1(self._client.token,
                                     client_secret=self._client.secret,
@@ -262,30 +261,33 @@ def get_request_token(self, callback=None):
                            client_secret=self._client.secret,
                            callback_uri=callback,
                            signature_method=SIGNATURE_PLAINTEXT)
-            r = requests.post(urlparse.urljoin(self.url, 'oauth/request_token/'), auth=oauth)
+            r = requests.post(urlparse.urljoin(
+                self.url, 'oauth/request_token/'), auth=oauth)
             credentials = urlparse.parse_qs(r.text)
             self.__request = OAuthToken(credentials.get('oauth_token')[0],
-                                        credentials.get('oauth_token_secret')[0])
+                                        credentials.get(
+                                        'oauth_token_secret')[0])
             return self.__request
 
         def get_authorization_url(self, request=None):
             "The second step of the OAuth workflow."
             if request is None:
                 if not self.__request.is_valid():
-                    raise APIError('You must obtain a request token to request '
-                                   'and access token. Use get_request_token() '
-                                   'first.')
+                    raise APIError('You must obtain a request token to'
+                                   'request and access token. Use'
+                                   'get_request_token() first.')
                 request = self.__request
             url = urlparse.urljoin(self.url, 'oauth/authorize/')
-            return url + '?' + urllib.urlencode(dict(oauth_token=request.token))
+            return url + '?' + urllib.urlencode(
+                dict(oauth_token=request.token))
 
         def get_access_token(self, request=None, verifier=None):
-            """The final step of the OAuth workflow. After this the client can make
-            API calls."""
+            """The final step of the OAuth workflow. After this the client
+            can make API calls."""
             if request is None:
                 if not self.__request.is_valid():
-                    raise APIError('You must obtain a request token to request '
-                                   'and access token. Use get_request_token() '
+                    raise APIError('You must obtain a request token to request'
+                                   'and access token. Use get_request_token()'
                                    'first.')
                 request = self.__request
             oauth = OAuth1(self._client.token,
@@ -294,7 +296,8 @@ def get_access_token(self, request=None, verifier=None):
                            resource_owner_secret=request.secret,
                            verifier=verifier,
                            signature_method=SIGNATURE_PLAINTEXT)
-            r = requests.post(urlparse.urljoin(self.url, 'oauth/access_token/'), auth=oauth)
+            r = requests.post(urlparse.urljoin(
+                self.url, 'oauth/access_token/'), auth=oauth)
             credentials = urlparse.parse_qs(r.text)
             self._access = OAuthToken(credentials.get('oauth_token')[0],
                                       credentials.get('oauth_token_secret')[0])
diff --git a/smartfile/errors.py b/smartfile/errors.py
index 758844a..5245ccd 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -33,7 +33,7 @@ def __init__(self, response, *args, **kwargs):
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']
             else:
-                self.detail = json['detail']
+                self.detail = json['src'][0]
         super(ResponseError, self).__init__(*args, **kwargs)
 
     def __str__(self):
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 1cb885a..be4989d 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -8,42 +8,43 @@
 
 if API_KEY is None:
     raise RuntimeError("API_KEY is required")
-    
+
 if API_PASSWORD is None:
     raise RuntimeError("API_PASSWORD is required")
-    
 
-class CustomOperationsTestCase(unittest.TestCase): 
-    
+
+class CustomOperationsTestCase(unittest.TestCase):
+
     def setUp(self):
         self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
         self.api = BasicClient(API_KEY, API_PASSWORD)
-        self.txtfile = self.current_dir + "resources/myfile.txt"
+        self.txtfile = self.current_dir + "../myfile.txt"
         self.uploaddata = None
-    
+
     def get_data(self):
         self.uploaddata = self.api.get("/path/info/myfile.txt")
         return self.uploaddata
-        
+
     def upload(self):
         data = open(self.txtfile, "rb")
         newfile = ('myfile.txt', data)
         self.api.upload(newfile)
-        self.assertEquals(self.get_data()['size'], os.path.getsize(self.txtfile))
-        
-    def download(self): 
+        self.assertEquals(self.get_data()['size'],
+                          os.path.getsize(self.txtfile))
+
+    def download(self):
         f = self.api.download("myfile.txt")
         self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
-    
+
     def move(self):
         self.api.move('myfile.txt', '/newFolder/')
-        
+
     def delete(self):
         self.api.delete("/newFolder/myfile.txt")
-        self.assertRaises(Exception, BasicClient.delete) 
-        
+        self.assertRaises(Exception, BasicClient.delete)
+
     def test_upload_download_move_delete_clean_up(self):
         self.upload()
         self.download()
         self.move()
-        self.delete()
\ No newline at end of file
+        self.delete()

From 16359a31904afd05bfa1ed206b095071cba3a31d Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 25 May 2016 15:48:42 -0400
Subject: [PATCH 49/90] actually addressed all comments in PR this time. See
 #3685

---
 smartfile/__init__.py | 5 +----
 smartfile/errors.py   | 7 ++++++-
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 98d8a33..26dafbb 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -129,10 +129,7 @@ def post(self, endpoint, id=None, **kwargs):
         return self._request('post', endpoint, id=id, data=kwargs)
 
     def delete(self, deletefile):
-        try:
-            return self.post('/path/oper/remove', path=deletefile)
-        except KeyError:
-            raise Exception("Destination file does not exist")
+        return self.post('/path/oper/remove', path=deletefile)
 
     def upload(self, usrfile):
         if usrfile[0].endswith('/'):
diff --git a/smartfile/errors.py b/smartfile/errors.py
index 5245ccd..a6eaf4c 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -33,7 +33,12 @@ def __init__(self, response, *args, **kwargs):
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']
             else:
-                self.detail = json['src'][0]
+                try:
+                    # A faulty move request returns the below response
+                    self.detail = json['src'][0]
+                except KeyError:
+                    # A faulty delete request returns the below response
+                    self.detail = json['path'][0]
         super(ResponseError, self).__init__(*args, **kwargs)
 
     def __str__(self):

From 9f0bd3dd3568271fed96de82c93ee9161dd18dcc Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 25 May 2016 16:04:25 -0400
Subject: [PATCH 50/90] update README to reflect changes

---
 README.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index e37ee1c..cc98ba4 100644
--- a/README.rst
+++ b/README.rst
@@ -203,8 +203,8 @@ To upload a file, pass a tuple of (filename, fileobject). For example:
 
 .. code:: python
 
-    >>> data = file('Song.mp3', 'rb')
     >>> from smartfile import BasicClient
+    >>> data = file('Song.mp3', 'rb')
     >>> api = BasicClient()
     >>> f = ('Song.mp3', data)
     >>> api.upload(f)

From 8c58ebfe89ff1d03ca194117ddc31a45f0521f91 Mon Sep 17 00:00:00 2001
From: Giampaolo Rodola <g.rodola@gmail.com>
Date: Wed, 25 May 2016 17:10:05 -0400
Subject: [PATCH 51/90] update upload() signature so that it accepts a filename
 and a fileobj as params

---
 smartfile/__init__.py  |  7 ++++---
 test/test_smartfile.py | 42 +++++++++++-------------------------------
 2 files changed, 15 insertions(+), 34 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 26dafbb..d7b1f14 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -131,10 +131,11 @@ def post(self, endpoint, id=None, **kwargs):
     def delete(self, deletefile):
         return self.post('/path/oper/remove', path=deletefile)
 
-    def upload(self, usrfile):
-        if usrfile[0].endswith('/'):
+    def upload(self, filename, fileobj):
+        if filename.endswith('/'):
             raise ValueError("File name should have no trailing slash")
-        return self.post('/path/data/', file=usrfile)
+        arg = (filename, fileobj)
+        return self.post('/path/data/', file=arg)
 
     def download(self, file_to_be_downloaded):
         """ file_to_be_downloaded is a file-like object that has already
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index be4989d..c7d3f4e 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -1,10 +1,12 @@
 import os
 import unittest
+from cStringIO import StringIO
 
 from smartfile import BasicClient
 
 API_KEY = os.environ.get("API_KEY")
 API_PASSWORD = os.environ.get("API_PASSWORD")
+TESTFN = "testfn"
 
 if API_KEY is None:
     raise RuntimeError("API_KEY is required")
@@ -16,35 +18,13 @@
 class CustomOperationsTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
         self.api = BasicClient(API_KEY, API_PASSWORD)
-        self.txtfile = self.current_dir + "../myfile.txt"
-        self.uploaddata = None
-
-    def get_data(self):
-        self.uploaddata = self.api.get("/path/info/myfile.txt")
-        return self.uploaddata
-
-    def upload(self):
-        data = open(self.txtfile, "rb")
-        newfile = ('myfile.txt', data)
-        self.api.upload(newfile)
-        self.assertEquals(self.get_data()['size'],
-                          os.path.getsize(self.txtfile))
-
-    def download(self):
-        f = self.api.download("myfile.txt")
-        self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
-
-    def move(self):
-        self.api.move('myfile.txt', '/newFolder/')
-
-    def delete(self):
-        self.api.delete("/newFolder/myfile.txt")
-        self.assertRaises(Exception, BasicClient.delete)
-
-    def test_upload_download_move_delete_clean_up(self):
-        self.upload()
-        self.download()
-        self.move()
-        self.delete()
+
+    def test_upload_and_download(self):
+        # Upload a file, download it, make sure the downloaded version
+        # has the same content.
+        f = StringIO('hello there')
+        f.seek(0)
+        self.api.upload(TESTFN, f)
+        r = self.api.download(TESTFN)
+        self.assertEqual(r.data, 'hello there')

From edac0241fd13b257f101eb316b823e2950fd15b2 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 25 May 2016 17:33:09 -0400
Subject: [PATCH 52/90] response to comments on PR

---
 .gitignore                    |  2 +-
 smartfile/.vscode/launch.json | 63 -----------------------------------
 smartfile/__init__.py         |  4 +--
 test/test_smartfile.py        |  5 +--
 4 files changed, 6 insertions(+), 68 deletions(-)
 delete mode 100644 smartfile/.vscode/launch.json

diff --git a/.gitignore b/.gitignore
index d9b067b..84b8b3c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,4 +90,4 @@ ENV/
 
 main.py
 
-/test/resources
\ No newline at end of file
+/test/resources
diff --git a/smartfile/.vscode/launch.json b/smartfile/.vscode/launch.json
deleted file mode 100644
index f93b136..0000000
--- a/smartfile/.vscode/launch.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Python",
-            "type": "python",
-            "request": "launch",
-            "stopOnEntry": true,
-            "program": "${file}",
-            "debugOptions": [
-                "WaitOnAbnormalExit",
-                "WaitOnNormalExit",
-                "RedirectOutput"
-            ]
-        },
-        {
-            "name": "Python Console App",
-            "type": "python",
-            "request": "launch",
-            "stopOnEntry": true,
-            "program": "${file}",
-            "externalConsole": true,
-            "debugOptions": [
-                "WaitOnAbnormalExit",
-                "WaitOnNormalExit"
-            ]
-        },
-        {
-            "name": "Django",
-            "type": "python",
-            "request": "launch",
-            "stopOnEntry": true,
-            "program": "${workspaceRoot}/manage.py",
-            "args": [
-                "runserver",
-                "--noreload"
-            ],
-            "debugOptions": [
-                "WaitOnAbnormalExit",
-                "WaitOnNormalExit",
-                "RedirectOutput",
-                "DjangoDebugging"
-            ]
-        },
-        {
-            "name": "Watson",
-            "type": "python",
-            "request": "launch",
-            "stopOnEntry": true,
-            "program": "${workspaceRoot}/console.py",
-            "args": [
-                "dev",
-                "runserver",
-                "--noreload=True"
-            ],
-            "debugOptions": [
-                "WaitOnAbnormalExit",
-                "WaitOnNormalExit",
-                "RedirectOutput"
-            ]
-        }
-    ]
-}
\ No newline at end of file
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index d7b1f14..c5a8f1e 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -144,10 +144,10 @@ def download(self, file_to_be_downloaded):
 
     def move(self, sourcefile, destination):
         # check destination folder for / at end
-        if destination.endswith("/"):
+        if not destination.endswith("/"):
             destination = destination + "/"
         # check destination folder for / at begining
-        if destination.startswith("/"):
+        if not destination.startswith("/"):
             destination = "/" + destination
         t = self.post('/path/oper/move/', src=sourcefile, dst=destination)
         while True:
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index c7d3f4e..fb22057 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -23,8 +23,9 @@ def setUp(self):
     def test_upload_and_download(self):
         # Upload a file, download it, make sure the downloaded version
         # has the same content.
-        f = StringIO('hello there')
+        file_contents = "hello there"
+        f = StringIO(file_contents)
         f.seek(0)
         self.api.upload(TESTFN, f)
         r = self.api.download(TESTFN)
-        self.assertEqual(r.data, 'hello there')
+        self.assertEqual(r.data, file_contents)

From a1bc2af5e2715af9303f383210d130ef16135182 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Thu, 26 May 2016 09:50:15 -0400
Subject: [PATCH 53/90] Updated README to reflect changes

---
 README.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index cc98ba4..80a78d6 100644
--- a/README.rst
+++ b/README.rst
@@ -199,15 +199,14 @@ File transfers
 
 Uploading and downloading files is supported.
 
-To upload a file, pass a tuple of (filename, fileobject). For example:
+To upload a file:
 
 .. code:: python
 
     >>> from smartfile import BasicClient
-    >>> data = file('Song.mp3', 'rb')
     >>> api = BasicClient()
-    >>> f = ('Song.mp3', data)
-    >>> api.upload(f)
+    >>> data = file('Song.mp3', 'rb')
+    >>> f = ("Song.mp3", data)
 
 
 Downloading is automatic, if the ``'Content-Type'`` header indicates

From 24fc7449e58efbd7324e813a3e8ba2ec63043edd Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Thu, 26 May 2016 11:40:08 -0400
Subject: [PATCH 54/90] update download

---
 smartfile/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index c5a8f1e..7eea49d 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -140,6 +140,8 @@ def upload(self, filename, fileobj):
     def download(self, file_to_be_downloaded):
         """ file_to_be_downloaded is a file-like object that has already
         been uploaded, you cannot download folders """
+        # no need to change download because it uses shutil.copyfileobj to
+        # download, which copies the data in chunks
         return self.get('/path/data/', file_to_be_downloaded)
 
     def move(self, sourcefile, destination):

From 1f5ddcee66819f9315de68508c493d4d360c403b Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Thu, 26 May 2016 13:59:50 -0400
Subject: [PATCH 55/90] updated download, updated requirements

---
 requirements.txt      |  1 +
 setup.py              |  3 ++-
 smartfile/__init__.py |  9 ++++++---
 smartfile/errors.py   |  1 +
 tests.py              | 22 ++++++++++++++--------
 5 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 3faf34c..a5438b9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,3 +5,4 @@ requests_oauthlib
 python-librsync
 coveralls
 coverage
+requests-toolbelt
diff --git a/setup.py b/setup.py
index 502523e..c92e624 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,7 @@
 
 import os
 import re
-#from distutils.core import setup
+# from distutils.core import setup
 from setuptools import setup
 
 VERSION_PATTERN = re.compile(r'^[^#]*__version__\W*\=\W*["\'](.*)["\']')
@@ -40,6 +40,7 @@ def get_path(path):
         'oauthlib',
         'requests',
         'requests_oauthlib',
+        'requests-toolbelt'
     ],
     author='SmartFile',
     author_email='tech@smartfile.com',
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 7eea49d..c9489cd 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -1,8 +1,10 @@
-from netrc import netrc
 import os
 import re
+import shutil
 import time
 import urllib
+
+from netrc import netrc
 try:
     import urlparse
     # Fixed pyflakes warning...
@@ -10,7 +12,6 @@
 except ImportError:
     from urllib import parse as urlparse
 
-
 import requests
 from requests.exceptions import RequestException
 
@@ -142,7 +143,9 @@ def download(self, file_to_be_downloaded):
         been uploaded, you cannot download folders """
         # no need to change download because it uses shutil.copyfileobj to
         # download, which copies the data in chunks
-        return self.get('/path/data/', file_to_be_downloaded)
+        o = file(file_to_be_downloaded, 'wb')
+        return shutil.copyfileobj(self.get('/path/data/',
+                                  file_to_be_downloaded), o)
 
     def move(self, sourcefile, destination):
         # check destination folder for / at end
diff --git a/smartfile/errors.py b/smartfile/errors.py
index a6eaf4c..239b2dc 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -29,6 +29,7 @@ def __init__(self, response, *args, **kwargs):
                 self.detail = six.u('Invalid URL, check your API path')
             else:
                 self.detail = six.u('Server error; check response for errors')
+                print self.response.text
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']
diff --git a/tests.py b/tests.py
index dc64db8..5512a32 100644
--- a/tests.py
+++ b/tests.py
@@ -47,9 +47,12 @@ def __init__(self, *args, **kwargs):
         BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def record(self, method, path, query=None, data=None):
-        request = TestHTTPRequestHandler.TestRequest(method, path, query=query,
+        request = TestHTTPRequestHandler.TestRequest(method,
+                                                     path,
+                                                     query=query,
                                                      data=data,
-                                                     headers=dict(self.headers.items()))
+                                                     headers=dict(
+                                                        self.headers.items()))
         self.server.requests.append(request)
         return request
 
@@ -66,7 +69,8 @@ def parse_and_record(self, method):
             l = int(self.headers['Content-Length'])
             ct, params = cgi.parse_header(self.headers['Content-Type'])
             if ct == 'multipart/form-data':
-                data = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={'REQUEST_METHOD': 'POST'})
+                data = cgi.FieldStorage(fp=self.rfile, headers=self.headers,
+                                        environ={'REQUEST_METHOD': 'POST'})
             else:
                 data = urlparse.parse_qs(self.rfile.read(l))
         request = self.record(method, urlp.path, query=query, data=data)
@@ -150,7 +154,7 @@ def assertData(self, key, value, request=-1):
 
     def assertIn(self, test_value, expected_set):
         msg = "%s did not occur in %s" % (test_value, expected_set)
-        self.assert_(test_value in expected_set, msg) 
+        self.assert_(test_value in expected_set, msg)
 
 
 class ClientTestCase(TestServerTestCase):
@@ -208,7 +212,8 @@ def test_call_is_GET(self):
         self.assertMethod('GET')
 
     def test_post_is_POST(self):
-        self.client.post('/user', username='bobafett', email='bobafett@example.com')
+        self.client.post('/user', username='bobafett',
+                         email='bobafett@example.com')
         self.assertMethod('POST')
 
     def test_get_is_GET(self):
@@ -314,7 +319,8 @@ def test_netrc(self):
 class OAuthClientTestCase(DownloadTestCase, UploadTestCase, MethodTestCase,
                           UrlGenerationTestCase, OAuthTestCase):
     def test_blank_client_token(self):
-        self.assertRaises(APIError, self.getClient, client_token='', client_secret='')
+        self.assertRaises(APIError, self.getClient,
+                          client_token='', client_secret='')
 
     def test_blank_access_token(self):
         client = self.getClient(access_token='', access_secret='')
@@ -350,7 +356,7 @@ def respond(self, request):
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
-        self.wfile.write(b"%s" % (json.dumps({ 'foo': 'bar' }), ))
+        self.wfile.write(b"%s" % (json.dumps({'foo': 'bar'}), ))
 
 
 class JSONTestCase(object):
@@ -359,7 +365,7 @@ class JSONTestCase(object):
     def test_throttle_GET(self):
         r = self.client.get('/user')
         self.assertMethod('GET')
-        self.assertEqual(r, { 'foo': 'bar' })
+        self.assertEqual(r, {'foo': 'bar'})
 
 
 class BasicJSONTestCase(JSONTestCase, BasicTestCase):

From 86b5d1da3dc7c96fd570a3bdaeb45c53f8a46755 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 11:49:36 -0400
Subject: [PATCH 56/90] reverted upload and tests

---
 setup.py               |  1 -
 smartfile/__init__.py  |  4 ++--
 test/test_smartfile.py | 43 ++++++++++++++++++++++++++++++------------
 3 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/setup.py b/setup.py
index c92e624..3d885f4 100644
--- a/setup.py
+++ b/setup.py
@@ -40,7 +40,6 @@ def get_path(path):
         'oauthlib',
         'requests',
         'requests_oauthlib',
-        'requests-toolbelt'
     ],
     author='SmartFile',
     author_email='tech@smartfile.com',
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index c9489cd..3c54de5 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -141,8 +141,8 @@ def upload(self, filename, fileobj):
     def download(self, file_to_be_downloaded):
         """ file_to_be_downloaded is a file-like object that has already
         been uploaded, you cannot download folders """
-        # no need to change download because it uses shutil.copyfileobj to
-        # download, which copies the data in chunks
+        # download uses shutil.copyfileobj to download, which copies
+        # the data in chunks
         o = file(file_to_be_downloaded, 'wb')
         return shutil.copyfileobj(self.get('/path/data/',
                                   file_to_be_downloaded), o)
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index fb22057..788f74b 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -1,12 +1,10 @@
 import os
 import unittest
-from cStringIO import StringIO
 
 from smartfile import BasicClient
 
 API_KEY = os.environ.get("API_KEY")
 API_PASSWORD = os.environ.get("API_PASSWORD")
-TESTFN = "testfn"
 
 if API_KEY is None:
     raise RuntimeError("API_KEY is required")
@@ -18,14 +16,35 @@
 class CustomOperationsTestCase(unittest.TestCase):
 
     def setUp(self):
+        self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
         self.api = BasicClient(API_KEY, API_PASSWORD)
-
-    def test_upload_and_download(self):
-        # Upload a file, download it, make sure the downloaded version
-        # has the same content.
-        file_contents = "hello there"
-        f = StringIO(file_contents)
-        f.seek(0)
-        self.api.upload(TESTFN, f)
-        r = self.api.download(TESTFN)
-        self.assertEqual(r.data, file_contents)
+        self.txtfile = self.current_dir + "myfile.txt"
+        self.uploaddata = None
+
+    def get_data(self):
+        self.uploaddata = self.api.get("/path/info/myfile.txt")
+        return self.uploaddata
+
+    def upload(self):
+        data = open(self.txtfile, "rb")
+        self.api.upload('myfile.txt', data)
+        self.assertEquals(self.get_data()['size'],
+                          os.path.getsize(self.txtfile))
+
+    def download(self):
+        self.api.download("myfile.txt")
+        f = open('myfile.txt', 'rb')
+        self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
+
+    def move(self):
+        self.api.move('myfile.txt', '/newFolder/')
+
+    def delete(self):
+        self.api.delete("/newFolder/myfile.txt")
+        self.assertRaises(Exception, BasicClient.delete)
+
+    def test_upload_download_move_delete(self):
+        self.upload()
+        self.download()
+        self.move()
+        self.delete()

From 5bb2c40dd253a80d01c24daf59603bda393fac46 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 11:52:06 -0400
Subject: [PATCH 57/90] update README to reflect changes

---
 README.rst | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index 80a78d6..2449cd5 100644
--- a/README.rst
+++ b/README.rst
@@ -215,12 +215,9 @@ returned.
 
 .. code:: python
 
-    >>> import shutil
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
     >>> f = api.download('foobar.png')
-    >>> with file('foobar.png', 'wb') as o:
-    >>>     shutil.copyfileobj(f, o)
 
 
 Tasks
@@ -236,7 +233,7 @@ Move files
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> api.move('/foobar.png', '/Folder/')
+    >>> api.move('myfile.txt', '/Folder/')
 
 
 Delete files
@@ -245,7 +242,7 @@ Delete files
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> api.delete('/foobar.png')
+    >>> api.delete('foobar.png')
 
 .. _SmartFile: http://www.smartfile.com/
 .. _Read more: http://www.smartfile.com/open-source.html

From d0771c6a1136e2fbd558f32fc29913ec779f241c Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 12:02:14 -0400
Subject: [PATCH 58/90] update requirements

---
 requirements.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index a5438b9..3faf34c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,4 +5,3 @@ requests_oauthlib
 python-librsync
 coveralls
 coverage
-requests-toolbelt

From 8fca6e7b908394dd5ad91a12aac3ce4767ceb4d6 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 12:47:01 -0400
Subject: [PATCH 59/90] remove debug code

---
 smartfile/errors.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/smartfile/errors.py b/smartfile/errors.py
index 239b2dc..a6eaf4c 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -29,7 +29,6 @@ def __init__(self, response, *args, **kwargs):
                 self.detail = six.u('Invalid URL, check your API path')
             else:
                 self.detail = six.u('Server error; check response for errors')
-                print self.response.text
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']

From 67bc0242b4d147e35cb4be288ed6b9f442726510 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 12:48:17 -0400
Subject: [PATCH 60/90] remove debug code

---
 smartfile/errors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/smartfile/errors.py b/smartfile/errors.py
index a6eaf4c..aee4dd1 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -28,7 +28,7 @@ def __init__(self, response, *args, **kwargs):
             if self.status_code == 404:
                 self.detail = six.u('Invalid URL, check your API path')
             else:
-                self.detail = six.u('Server error; check response for errors')
+                self.detail = six.u('Server error; check response for error')
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']

From 6220f7cb9a949608a987f836c3c4795f8460947d Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 12:50:13 -0400
Subject: [PATCH 61/90] remove debug code

---
 smartfile/errors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/smartfile/errors.py b/smartfile/errors.py
index aee4dd1..a6eaf4c 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -28,7 +28,7 @@ def __init__(self, response, *args, **kwargs):
             if self.status_code == 404:
                 self.detail = six.u('Invalid URL, check your API path')
             else:
-                self.detail = six.u('Server error; check response for error')
+                self.detail = six.u('Server error; check response for errors')
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']

From d56e337f9badf6e07ab0d9cdcc6cd55820b8abf1 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 12:52:41 -0400
Subject: [PATCH 62/90] remove debug code

---
 smartfile/errors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/smartfile/errors.py b/smartfile/errors.py
index a6eaf4c..aee4dd1 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -28,7 +28,7 @@ def __init__(self, response, *args, **kwargs):
             if self.status_code == 404:
                 self.detail = six.u('Invalid URL, check your API path')
             else:
-                self.detail = six.u('Server error; check response for errors')
+                self.detail = six.u('Server error; check response for error')
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']

From 9fce070e976256ae15bea2b74ff73d29d66a2942 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 27 May 2016 13:02:04 -0400
Subject: [PATCH 63/90] fixed python3 unsupported call

---
 smartfile/__init__.py | 2 +-
 smartfile/errors.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 3c54de5..e889e4c 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -143,7 +143,7 @@ def download(self, file_to_be_downloaded):
         been uploaded, you cannot download folders """
         # download uses shutil.copyfileobj to download, which copies
         # the data in chunks
-        o = file(file_to_be_downloaded, 'wb')
+        o = open(file_to_be_downloaded, 'wb')
         return shutil.copyfileobj(self.get('/path/data/',
                                   file_to_be_downloaded), o)
 
diff --git a/smartfile/errors.py b/smartfile/errors.py
index aee4dd1..a6eaf4c 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -28,7 +28,7 @@ def __init__(self, response, *args, **kwargs):
             if self.status_code == 404:
                 self.detail = six.u('Invalid URL, check your API path')
             else:
-                self.detail = six.u('Server error; check response for error')
+                self.detail = six.u('Server error; check response for errors')
         else:
             if self.status_code == 400 and 'field_errors' in json:
                 self.detail = json['field_errors']

From 06cae98d6525b4e0dfd825b2b6b18f4a6e396470 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 8 Jun 2016 09:03:20 -0400
Subject: [PATCH 64/90] update README

---
 README.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 2449cd5..6a5b8a3 100644
--- a/README.rst
+++ b/README.rst
@@ -205,8 +205,8 @@ To upload a file:
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> data = file('Song.mp3', 'rb')
-    >>> f = ("Song.mp3", data)
+    >>> data = file('test.txt', 'rb')
+    >>> api.upload('test.txt', data)
 
 
 Downloading is automatic, if the ``'Content-Type'`` header indicates
@@ -217,7 +217,7 @@ returned.
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> f = api.download('foobar.png')
+    >>> api.download('foobar.png')
 
 
 Tasks

From e5da9edabd49f0b8aebae6bd51a8d38657c61de3 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Mon, 27 Jun 2016 13:20:45 -0400
Subject: [PATCH 65/90] update README, rename arguments

---
 README.rst            |  2 +-
 smartfile/__init__.py | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/README.rst b/README.rst
index 6a5b8a3..69bce94 100644
--- a/README.rst
+++ b/README.rst
@@ -205,7 +205,7 @@ To upload a file:
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> data = file('test.txt', 'rb')
+    >>> data = open('test.txt', 'rb')
     >>> api.upload('test.txt', data)
 
 
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index e889e4c..d1b627a 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -147,14 +147,14 @@ def download(self, file_to_be_downloaded):
         return shutil.copyfileobj(self.get('/path/data/',
                                   file_to_be_downloaded), o)
 
-    def move(self, sourcefile, destination):
+    def move(self, src_path, dst_path):
         # check destination folder for / at end
-        if not destination.endswith("/"):
-            destination = destination + "/"
+        if not dst_path.endswith("/"):
+            dst_path = dst_path + "/"
         # check destination folder for / at begining
-        if not destination.startswith("/"):
-            destination = "/" + destination
-        t = self.post('/path/oper/move/', src=sourcefile, dst=destination)
+        if not dst_path.startswith("/"):
+            dst_path = "/" + dst_path
+        t = self.post('/path/oper/move/', src=src_path, dst=dst_path)
         while True:
             s = self.get('/task', t['uuid'])
             if s['result']['status'] == 'SUCCESS':

From e38550a2d6a83c9c952c212cd38559d997574214 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Mon, 27 Jun 2016 14:48:34 -0400
Subject: [PATCH 66/90] update move function

---
 README.rst            | 10 ++++++++++
 smartfile/__init__.py |  6 +-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/README.rst b/README.rst
index 69bce94..1797448 100644
--- a/README.rst
+++ b/README.rst
@@ -234,6 +234,16 @@ Move files
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
     >>> api.move('myfile.txt', '/Folder/')
+    >>> while True:
+    >>>     try:
+    >>>         s = self.get('/task', api['uuid'])
+    >>>         # Sleep to assure the server doesn't get overloaded
+    >>>         time.sleep(1)
+    >>>         if s['result']['status'] == 'SUCCESS':
+    >>>             break
+    >>>     except Exception as e:
+    >>>         print e
+    >>>         break
 
 
 Delete files
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index d1b627a..48267fc 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -154,11 +154,7 @@ def move(self, src_path, dst_path):
         # check destination folder for / at begining
         if not dst_path.startswith("/"):
             dst_path = "/" + dst_path
-        t = self.post('/path/oper/move/', src=src_path, dst=dst_path)
-        while True:
-            s = self.get('/task', t['uuid'])
-            if s['result']['status'] == 'SUCCESS':
-                break
+        return self.post('/path/oper/move/', src=src_path, dst=dst_path)
 
 
 class BasicClient(Client):

From 1cb50ec31e8c287bc75f23846035b95a5c43037f Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Mon, 27 Jun 2016 15:28:25 -0400
Subject: [PATCH 67/90] update README

---
 README.rst | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 1797448..89d847b 100644
--- a/README.rst
+++ b/README.rst
@@ -231,16 +231,25 @@ Move files
 
 .. code:: python
 
+    >>> import logging
     >>> from smartfile import BasicClient
+    >>>
     >>> api = BasicClient()
-    >>> api.move('myfile.txt', '/Folder/')
+    >>>
+    >>> LOGGER = logging.getLogger(__name__)
+    >>> LOGGER.setLevel(logging.INFO)
+    >>>
+    >>> api.move('file.txt', '/newFolder')
+    >>>
     >>> while True:
     >>>     try:
-    >>>         s = self.get('/task', api['uuid'])
+    >>>         s = api.get('/task', api['uuid'])
     >>>         # Sleep to assure the server doesn't get overloaded
     >>>         time.sleep(1)
     >>>         if s['result']['status'] == 'SUCCESS':
     >>>             break
+    >>>         elif s['result']['status'] == 'FAILURE':
+    >>>             LOGGER.info("Task failure: " + s['uuid'])
     >>>     except Exception as e:
     >>>         print e
     >>>         break

From e4a47dd31ccd2285fa7c4d149502a9436fa0366d Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 28 Jun 2016 09:08:33 -0400
Subject: [PATCH 68/90] addressed comments on PR

---
 README.rst            | 4 ++--
 smartfile/__init__.py | 8 +++++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 89d847b..baae7e9 100644
--- a/README.rst
+++ b/README.rst
@@ -205,8 +205,8 @@ To upload a file:
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> data = open('test.txt', 'rb')
-    >>> api.upload('test.txt', data)
+    >>> file = open('test.txt', 'rb')
+    >>> api.upload('test.txt', file)
 
 
 Downloading is automatic, if the ``'Content-Type'`` header indicates
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 48267fc..aaee056 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -134,7 +134,7 @@ def delete(self, deletefile):
 
     def upload(self, filename, fileobj):
         if filename.endswith('/'):
-            raise ValueError("File name should have no trailing slash")
+            filename = filename[:-1]
         arg = (filename, fileobj)
         return self.post('/path/data/', file=arg)
 
@@ -154,6 +154,12 @@ def move(self, src_path, dst_path):
         # check destination folder for / at begining
         if not dst_path.startswith("/"):
             dst_path = "/" + dst_path
+        # check destination folder for / at end
+        if not src_path.endswith("/"):
+            src_path = src_path + "/"
+        # check destination folder for / at begining
+        if not src_path.startswith("/"):
+            src_path = "/" + src_path
         return self.post('/path/oper/move/', src=src_path, dst=dst_path)
 
 

From 733db726224dbd07681428ca9908fa6c96baa2cc Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Tue, 28 Jun 2016 15:21:23 -0400
Subject: [PATCH 69/90] response to PR comments

---
 README.rst             |  2 +-
 smartfile/__init__.py  | 23 +++++++++++++++--------
 test/test_smartfile.py | 14 ++++++++------
 tests.py               | 15 +++++++++------
 4 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/README.rst b/README.rst
index baae7e9..3d2e804 100644
--- a/README.rst
+++ b/README.rst
@@ -261,7 +261,7 @@ Delete files
 
     >>> from smartfile import BasicClient
     >>> api = BasicClient()
-    >>> api.delete('foobar.png')
+    >>> api.remove('foobar.png')
 
 .. _SmartFile: http://www.smartfile.com/
 .. _Read more: http://www.smartfile.com/open-source.html
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index aaee056..0421154 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -129,8 +129,15 @@ def put(self, endpoint, id=None, **kwargs):
     def post(self, endpoint, id=None, **kwargs):
         return self._request('post', endpoint, id=id, data=kwargs)
 
-    def delete(self, deletefile):
-        return self.post('/path/oper/remove', path=deletefile)
+    def delete(self, endpoint, id=None, **kwargs):
+        return self._request('delete', endpoint, id=id, data=kwargs)
+
+    def remove(self, deletefile):
+        try:
+            return self.post('/path/oper/remove', path=deletefile)
+        except KeyError:
+            raise Exception("Destination file does not exist")
+        # return self.post('/path/oper/remove', path=deletefile)
 
     def upload(self, filename, fileobj):
         if filename.endswith('/'):
@@ -149,18 +156,18 @@ def download(self, file_to_be_downloaded):
 
     def move(self, src_path, dst_path):
         # check destination folder for / at end
-        if not dst_path.endswith("/"):
-            dst_path = dst_path + "/"
-        # check destination folder for / at begining
-        if not dst_path.startswith("/"):
-            dst_path = "/" + dst_path
-        # check destination folder for / at end
         if not src_path.endswith("/"):
             src_path = src_path + "/"
         # check destination folder for / at begining
         if not src_path.startswith("/"):
             src_path = "/" + src_path
         return self.post('/path/oper/move/', src=src_path, dst=dst_path)
+        # check destination folder for / at end
+        if not dst_path.endswith("/"):
+            dst_path = dst_path + "/"
+        # check destination folder for / at begining
+        if not dst_path.startswith("/"):
+            dst_path = "/" + dst_path
 
 
 class BasicClient(Client):
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 788f74b..0c0ad78 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -2,6 +2,7 @@
 import unittest
 
 from smartfile import BasicClient
+from smartfile.errors import ResponseError
 
 API_KEY = os.environ.get("API_KEY")
 API_PASSWORD = os.environ.get("API_PASSWORD")
@@ -27,21 +28,22 @@ def get_data(self):
 
     def upload(self):
         data = open(self.txtfile, "rb")
-        self.api.upload('myfile.txt', data)
+        self.api.upload(self.txtfile, data)
         self.assertEquals(self.get_data()['size'],
                           os.path.getsize(self.txtfile))
 
     def download(self):
         self.api.download("myfile.txt")
-        f = open('myfile.txt', 'rb')
-        self.assertEquals(f.readlines(), open(self.txtfile, "rb").readlines())
+        self.assertEquals(os.path.getsize(self.txtfile),
+                          self.get_data()['size'])
 
     def move(self):
         self.api.move('myfile.txt', '/newFolder/')
 
-    def delete(self):
-        self.api.delete("/newFolder/myfile.txt")
-        self.assertRaises(Exception, BasicClient.delete)
+    def remove(self):
+        self.api.remove("/newFolder/myfile.txt")
+        with self.assertRaises(ResponseError):
+            self.api.remove("/newFolder/myfile.txt")
 
     def test_upload_download_move_delete(self):
         self.upload()
diff --git a/tests.py b/tests.py
index 5512a32..198915e 100644
--- a/tests.py
+++ b/tests.py
@@ -47,12 +47,15 @@ def __init__(self, *args, **kwargs):
         BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
 
     def record(self, method, path, query=None, data=None):
-        request = TestHTTPRequestHandler.TestRequest(method,
-                                                     path,
-                                                     query=query,
-                                                     data=data,
-                                                     headers=dict(
-                                                        self.headers.items()))
+        request = TestHTTPRequestHandler.TestRequest(
+            method,
+            path,
+            query=query,
+            data=data,
+            headers=dict(
+                self.headers.items()
+            )
+        )
         self.server.requests.append(request)
         return request
 

From 0ce17d3b0e4b5d905792a8f6456b0b216a0a3955 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 29 Jun 2016 09:37:36 -0400
Subject: [PATCH 70/90] update unit tests to have static file name

---
 README.rst             | 10 +++++++++
 test/test_smartfile.py | 47 ++++++++++++++++++++++++++++--------------
 2 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/README.rst b/README.rst
index 3d2e804..3766063 100644
--- a/README.rst
+++ b/README.rst
@@ -265,3 +265,13 @@ Delete files
 
 .. _SmartFile: http://www.smartfile.com/
 .. _Read more: http://www.smartfile.com/open-source.html
+
+
+
+Running the Tests
+--------------
+To run tests for the test.py file:
+`nosetests -v tests.py`
+
+To run tests for the test_smartfile.py file:
+`API_KEY='****' API_PASSWORD='****' nosetests test`
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 0c0ad78..d241e16 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -1,11 +1,13 @@
 import os
 import unittest
+from StringIO import StringIO
 
 from smartfile import BasicClient
 from smartfile.errors import ResponseError
 
 API_KEY = os.environ.get("API_KEY")
 API_PASSWORD = os.environ.get("API_PASSWORD")
+TESTFN = "testfn"
 
 if API_KEY is None:
     raise RuntimeError("API_KEY is required")
@@ -17,36 +19,49 @@
 class CustomOperationsTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
+        # self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
         self.api = BasicClient(API_KEY, API_PASSWORD)
-        self.txtfile = self.current_dir + "myfile.txt"
-        self.uploaddata = None
+        # Make directory for tests
+        self.api.post('/path/oper/mkdir/', path='/testfn2')
+        # self.txtfile = self.current_dir + "myfile.txt"
 
     def get_data(self):
-        self.uploaddata = self.api.get("/path/info/myfile.txt")
-        return self.uploaddata
+        data = self.api.get("/path/info/testfn")
+        return data
+
+    def tearDown(self):
+        self.api.remove('/testfn2')
+        os.remove('testfn')
 
     def upload(self):
-        data = open(self.txtfile, "rb")
-        self.api.upload(self.txtfile, data)
-        self.assertEquals(self.get_data()['size'],
-                          os.path.getsize(self.txtfile))
+        file_contents = "hello"
+        f = StringIO(file_contents)
+        f.seek(0)
+        self.api.upload(TESTFN, f)
+        self.assertEquals(self.get_data()['size'], f.len)
+        # data = open(self.txtfile, "rb")
+        # self.api.upload(self.txtfile, data)
+        # self.assertEquals(self.get_data()['size'],
+        #                   os.path.getsize(self.txtfile))
 
     def download(self):
-        self.api.download("myfile.txt")
-        self.assertEquals(os.path.getsize(self.txtfile),
-                          self.get_data()['size'])
+        self.api.download('testfn')
+        self.assertEquals(self.get_data()['size'], os.path.getsize('testfn'))
+        # self.api.download("myfile.txt")
+        # self.assertEquals(os.path.getsize(self.txtfile),
+        #                   self.get_data()['size'])
 
     def move(self):
-        self.api.move('myfile.txt', '/newFolder/')
+        self.api.move('testfn', '/testfn2/')
+        # self.api.move('myfile.txt', '/newFolder/')
 
     def remove(self):
-        self.api.remove("/newFolder/myfile.txt")
+        self.api.remove("/testfn2/testfn")
         with self.assertRaises(ResponseError):
-            self.api.remove("/newFolder/myfile.txt")
+            self.api.remove("/testfn2/testfn")
 
     def test_upload_download_move_delete(self):
         self.upload()
         self.download()
         self.move()
-        self.delete()
+        self.remove()

From 8869a7fe392c90b299610478d2822a222fb67c2f Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 29 Jun 2016 10:41:38 -0400
Subject: [PATCH 71/90] remove commented out code

---
 test/test_smartfile.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index d241e16..b943fe7 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -19,11 +19,9 @@
 class CustomOperationsTestCase(unittest.TestCase):
 
     def setUp(self):
-        # self.current_dir = os.path.dirname(os.path.realpath(__file__)) + "/"
         self.api = BasicClient(API_KEY, API_PASSWORD)
         # Make directory for tests
         self.api.post('/path/oper/mkdir/', path='/testfn2')
-        # self.txtfile = self.current_dir + "myfile.txt"
 
     def get_data(self):
         data = self.api.get("/path/info/testfn")
@@ -39,21 +37,13 @@ def upload(self):
         f.seek(0)
         self.api.upload(TESTFN, f)
         self.assertEquals(self.get_data()['size'], f.len)
-        # data = open(self.txtfile, "rb")
-        # self.api.upload(self.txtfile, data)
-        # self.assertEquals(self.get_data()['size'],
-        #                   os.path.getsize(self.txtfile))
 
     def download(self):
         self.api.download('testfn')
         self.assertEquals(self.get_data()['size'], os.path.getsize('testfn'))
-        # self.api.download("myfile.txt")
-        # self.assertEquals(os.path.getsize(self.txtfile),
-        #                   self.get_data()['size'])
 
     def move(self):
         self.api.move('testfn', '/testfn2/')
-        # self.api.move('myfile.txt', '/newFolder/')
 
     def remove(self):
         self.api.remove("/testfn2/testfn")

From 4a76291b581eb840790ac920f7b34cb08b8d737c Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Wed, 29 Jun 2016 10:45:38 -0400
Subject: [PATCH 72/90] update README

---
 README.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.rst b/README.rst
index 3766063..6b34961 100644
--- a/README.rst
+++ b/README.rst
@@ -268,10 +268,12 @@ Delete files
 
 
 
-Running the Tests
+Running Tests
 --------------
 To run tests for the test.py file:
-`nosetests -v tests.py`
+::
+    nosetests -v tests.py
 
 To run tests for the test_smartfile.py file:
-`API_KEY='****' API_PASSWORD='****' nosetests test`
+::
+    API_KEY='****' API_PASSWORD='****' nosetests test

From 10a93315780c3f9f8dc4ab177e49c43f72a53e08 Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Thu, 14 Jul 2016 15:15:22 -0400
Subject: [PATCH 73/90] fixed testing, removed code smell

---
 smartfile/__init__.py  |  3 +--
 test/test_smartfile.py | 18 ++++++++++--------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 0421154..dd3ff4f 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -137,7 +137,6 @@ def remove(self, deletefile):
             return self.post('/path/oper/remove', path=deletefile)
         except KeyError:
             raise Exception("Destination file does not exist")
-        # return self.post('/path/oper/remove', path=deletefile)
 
     def upload(self, filename, fileobj):
         if filename.endswith('/'):
@@ -161,13 +160,13 @@ def move(self, src_path, dst_path):
         # check destination folder for / at begining
         if not src_path.startswith("/"):
             src_path = "/" + src_path
-        return self.post('/path/oper/move/', src=src_path, dst=dst_path)
         # check destination folder for / at end
         if not dst_path.endswith("/"):
             dst_path = dst_path + "/"
         # check destination folder for / at begining
         if not dst_path.startswith("/"):
             dst_path = "/" + dst_path
+        return self.post('/path/oper/move/', src=src_path, dst=dst_path)
 
 
 class BasicClient(Client):
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index b943fe7..4bd7d27 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -7,7 +7,6 @@
 
 API_KEY = os.environ.get("API_KEY")
 API_PASSWORD = os.environ.get("API_PASSWORD")
-TESTFN = "testfn"
 
 if API_KEY is None:
     raise RuntimeError("API_KEY is required")
@@ -15,13 +14,17 @@
 if API_PASSWORD is None:
     raise RuntimeError("API_PASSWORD is required")
 
+TESTFN = "testfn"
+file_contents = "hello"
+TESTFN2 = "testfn2"
+
 
 class CustomOperationsTestCase(unittest.TestCase):
 
     def setUp(self):
         self.api = BasicClient(API_KEY, API_PASSWORD)
         # Make directory for tests
-        self.api.post('/path/oper/mkdir/', path='/testfn2')
+        self.api.post('/path/oper/mkdir/', path=TESTFN2)
 
     def get_data(self):
         data = self.api.get("/path/info/testfn")
@@ -32,23 +35,22 @@ def tearDown(self):
         os.remove('testfn')
 
     def upload(self):
-        file_contents = "hello"
         f = StringIO(file_contents)
         f.seek(0)
         self.api.upload(TESTFN, f)
         self.assertEquals(self.get_data()['size'], f.len)
 
     def download(self):
-        self.api.download('testfn')
-        self.assertEquals(self.get_data()['size'], os.path.getsize('testfn'))
+        self.api.download(TESTFN)
+        self.assertEquals(self.get_data()['size'], os.path.getsize(TESTFN))
 
     def move(self):
-        self.api.move('testfn', '/testfn2/')
+        self.api.move(TESTFN, TESTFN2)
 
     def remove(self):
-        self.api.remove("/testfn2/testfn")
+        self.api.remove(os.path.join(TESTFN2, TESTFN))
         with self.assertRaises(ResponseError):
-            self.api.remove("/testfn2/testfn")
+            self.api.remove(os.path.join(TESTFN2, TESTFN))
 
     def test_upload_download_move_delete(self):
         self.upload()

From 6e3bd2f5460049c9702bf44b37095c635ad8460b Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Thu, 14 Jul 2016 15:35:23 -0400
Subject: [PATCH 74/90] Fix error handling to catch if JSON is not returned

---
 smartfile/errors.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/smartfile/errors.py b/smartfile/errors.py
index a6eaf4c..c741929 100644
--- a/smartfile/errors.py
+++ b/smartfile/errors.py
@@ -38,7 +38,10 @@ def __init__(self, response, *args, **kwargs):
                     self.detail = json['src'][0]
                 except KeyError:
                     # A faulty delete request returns the below response
-                    self.detail = json['path'][0]
+                    try:
+                        self.detail = json['path'][0]
+                    except KeyError:
+                        self.detail = six.u('Error: %s' % response.content)
         super(ResponseError, self).__init__(*args, **kwargs)
 
     def __str__(self):

From c626a20fb104cf6d38c3faf3026d026102cabbbb Mon Sep 17 00:00:00 2001
From: Jennifer <jennifernicoleemery@gmail.com>
Date: Fri, 15 Jul 2016 10:43:24 -0400
Subject: [PATCH 75/90] implemented better verbiage, removed commented out
 code,

---
 README.rst            | 2 +-
 setup.py              | 1 -
 smartfile/__init__.py | 4 ++--
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index 6b34961..d5574e3 100644
--- a/README.rst
+++ b/README.rst
@@ -244,7 +244,7 @@ Move files
     >>> while True:
     >>>     try:
     >>>         s = api.get('/task', api['uuid'])
-    >>>         # Sleep to assure the server doesn't get overloaded
+    >>>         # Sleep to assure the user does not get rate limited
     >>>         time.sleep(1)
     >>>         if s['result']['status'] == 'SUCCESS':
     >>>             break
diff --git a/setup.py b/setup.py
index 3d885f4..cb12ad4 100644
--- a/setup.py
+++ b/setup.py
@@ -2,7 +2,6 @@
 
 import os
 import re
-# from distutils.core import setup
 from setuptools import setup
 
 VERSION_PATTERN = re.compile(r'^[^#]*__version__\W*\=\W*["\'](.*)["\']')
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index dd3ff4f..7d0dd88 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -297,8 +297,8 @@ def get_access_token(self, request=None, verifier=None):
             can make API calls."""
             if request is None:
                 if not self.__request.is_valid():
-                    raise APIError('You must obtain a request token to request'
-                                   'and access token. Use get_request_token()'
+                    raise APIError('You must obtain a request token to request '
+                                   'and access token. Use get_request_token() '
                                    'first.')
                 request = self.__request
             oauth = OAuth1(self._client.token,

From 3caa4716145ea96360e10895ed190ed16acda02f Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Thu, 1 Jun 2017 15:50:02 -0400
Subject: [PATCH 76/90] Use requirements.txt dependencies for setup.py

---
 MANIFEST.in |  1 +
 setup.py    | 11 ++++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 9561fb1..8e5e8a9 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1 +1,2 @@
+include requirements.txt
 include README.rst
diff --git a/setup.py b/setup.py
index cb12ad4..fd2e979 100644
--- a/setup.py
+++ b/setup.py
@@ -7,6 +7,11 @@
 VERSION_PATTERN = re.compile(r'^[^#]*__version__\W*\=\W*["\'](.*)["\']')
 VERSION = None
 
+with open('requirements.txt') as f:
+    required = f.read().splitlines()
+
+required = [r for r in required if not r.startswith('git')]
+
 
 def get_path(path):
     return os.path.join(os.path.dirname(__file__), path)
@@ -35,11 +40,7 @@ def get_path(path):
     version=versrel,
     description='A Python client for the SmartFile API.',
     long_description=long_description,
-    install_requires=[
-        'oauthlib',
-        'requests',
-        'requests_oauthlib',
-    ],
+    install_requires=required,
     author='SmartFile',
     author_email='tech@smartfile.com',
     maintainer='Ben Timby',

From ddde94213981da8f6230a51dbe5059d36e69578b Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@smartfile.com>
Date: Thu, 1 Jun 2017 15:54:41 -0400
Subject: [PATCH 77/90] Normalize version, 2.18 vs 2.1-post8.

---
 setup.py              | 4 +---
 smartfile/__init__.py | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/setup.py b/setup.py
index cb12ad4..17eaeb2 100644
--- a/setup.py
+++ b/setup.py
@@ -25,14 +25,12 @@ def get_path(path):
 
 
 name = 'smartfile'
-release = '7'
-versrel = VERSION + '-' + release
 long_description = open(get_path('README.rst'), 'r').read()
 
 
 setup(
     name=name,
-    version=versrel,
+    version=VERSION,
     description='A Python client for the SmartFile API.',
     long_description=long_description,
     install_requires=[
diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 7d0dd88..4764250 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -20,7 +20,7 @@
 from smartfile.errors import ResponseError
 
 
-__version__ = '2.1'
+__version__ = '2.18'
 
 API_URL = 'https://app.smartfile.com/'
 

From e3beeb98cbda493881b53d85cd05dd46efc502d8 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@smartfile.com>
Date: Thu, 1 Jun 2017 15:57:47 -0400
Subject: [PATCH 78/90] Removed unsupported pip option.

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b55be14..db08166 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,10 +6,10 @@ python:
 before_install:
   - sudo apt-get install librsync1 -qq
 install:
-  - pip install --timeout=30 pep8 --use-mirrors
-  - pip install --timeout=30 pyflakes --use-mirrors
-  - pip install --timeout=30 -r requirements.txt --use-mirrors
-  - pip install --timeout=30 -q -e . --use-mirrors
+  - pip install --timeout=30 pep8
+  - pip install --timeout=30 pyflakes
+  - pip install --timeout=30 -r requirements.txt
+  - pip install --timeout=30 -q -e .
 before_script:
   - make verify
 script:

From 9d2d6cf89c495ba36c99dca8a4d28fa0e0e5d697 Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Thu, 1 Jun 2017 16:01:02 -0400
Subject: [PATCH 79/90] use-mirrors is no longer a valid parameter

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b55be14..db08166 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,10 +6,10 @@ python:
 before_install:
   - sudo apt-get install librsync1 -qq
 install:
-  - pip install --timeout=30 pep8 --use-mirrors
-  - pip install --timeout=30 pyflakes --use-mirrors
-  - pip install --timeout=30 -r requirements.txt --use-mirrors
-  - pip install --timeout=30 -q -e . --use-mirrors
+  - pip install --timeout=30 pep8
+  - pip install --timeout=30 pyflakes
+  - pip install --timeout=30 -r requirements.txt
+  - pip install --timeout=30 -q -e .
 before_script:
   - make verify
 script:

From 7cf59fa7aa005a76420a971901a806b0739d79db Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@smartfile.com>
Date: Thu, 1 Jun 2017 16:02:36 -0400
Subject: [PATCH 80/90] Adjust Python versions

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index db08166..e086024 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,8 +1,8 @@
 language: python
 python:
-  - "2.6"
   - "2.7"
-  - "3.2"
+  - "3.3"
+  - "3.6"
 before_install:
   - sudo apt-get install librsync1 -qq
 install:

From 2a64f90f16f0690145e7c897a2648a4a513e37f2 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@smartfile.com>
Date: Thu, 1 Jun 2017 16:34:20 -0400
Subject: [PATCH 81/90] Use major version to determine URL (allowing minor
 revisions).

---
 smartfile/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 4764250..460a68c 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -21,6 +21,7 @@
 
 
 __version__ = '2.18'
+__major__ = __version__.split('.')[0]
 
 API_URL = 'https://app.smartfile.com/'
 
@@ -41,7 +42,7 @@ def clean_tokens(*args):
 
 class Client(object):
     """Base API client, handles communication, retry, versioning etc."""
-    def __init__(self, url=None, version=__version__, throttle_wait=True):
+    def __init__(self, url=None, version=__major__, throttle_wait=True):
         self.url = url or os.environ.get('SMARTFILE_API_URL') or API_URL
         self.version = version
         self.throttle_wait = throttle_wait

From ac8194608546678b584fc9b82bb707017075f0d4 Mon Sep 17 00:00:00 2001
From: Ben Timby <btimby@smartfile.com>
Date: Thu, 1 Jun 2017 16:40:31 -0400
Subject: [PATCH 82/90] Version bump for release.

---
 smartfile/__init__.py  | 2 +-
 test/test_smartfile.py | 8 +++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 460a68c..44f3b3d 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -20,7 +20,7 @@
 from smartfile.errors import ResponseError
 
 
-__version__ = '2.18'
+__version__ = '2.19'
 __major__ = __version__.split('.')[0]
 
 API_URL = 'https://app.smartfile.com/'
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 4bd7d27..f59fdd3 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -1,6 +1,12 @@
+from __future__ import absolute_import
+
 import os
 import unittest
-from StringIO import StringIO
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 from smartfile import BasicClient
 from smartfile.errors import ResponseError

From 13dfc05eafcb335c9283e1b0722a55603596d765 Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Mon, 5 Jun 2017 10:43:21 -0400
Subject: [PATCH 83/90] Set Makefile targets as PHONY

---
 Makefile | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/Makefile b/Makefile
index f53fc73..b34d529 100644
--- a/Makefile
+++ b/Makefile
@@ -1,23 +1,30 @@
+.PHONY: test
 test:
 	coverage run tests.py
 
+.PHONY: verify
 verify:
 	pyflakes smartfile
 	pep8 --ignore=E501,E225 smartfile
 
+.PHONY: install
 install:
 	python setup.py install
 
+.PHONY: publish
 publish:
 	python setup.py register
 	python setup.py sdist upload
 
+.PHONY: profile
 profile:
 	python profile.py
 
+.PHONY: clean
 clean:
 	find . -name *.pyc -delete
 
+.PHONY: distclean
 distclean: clean
 	rm -rf env
 

From 389d4da7c899a91809083baff7deb79069a10108 Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Mon, 5 Jun 2017 13:23:50 -0400
Subject: [PATCH 84/90] Fix string encoding

---
 tests.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests.py b/tests.py
index 198915e..58b7997 100644
--- a/tests.py
+++ b/tests.py
@@ -303,8 +303,8 @@ def test_netrc(self):
                     address, port = address
                 else:
                     port = self.server.server_port
-                netrc = b"machine 127.0.0.1:%s\n  login %s\n  password %s" % (
-                        port, API_KEY, API_PASSWORD)
+                netrc = b"machine 127.0.0.1:%i\n  login %s\n  password %s" % (
+                        port, API_KEY.encode(), API_PASSWORD.encode())
                 os.write(fd, netrc)
             finally:
                 os.close(fd)
@@ -359,7 +359,7 @@ def respond(self, request):
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
-        self.wfile.write(b"%s" % (json.dumps({'foo': 'bar'}), ))
+        self.wfile.write(b"%s" % (json.dumps({'foo': 'bar'}).encode(), ))
 
 
 class JSONTestCase(object):

From 968dc9d8f6a398d9703a40cdb9f6f77045e79d9e Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Mon, 5 Jun 2017 13:54:43 -0400
Subject: [PATCH 85/90] Specify encoding

---
 tests.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests.py b/tests.py
index 58b7997..241a70a 100644
--- a/tests.py
+++ b/tests.py
@@ -304,7 +304,8 @@ def test_netrc(self):
                 else:
                     port = self.server.server_port
                 netrc = b"machine 127.0.0.1:%i\n  login %s\n  password %s" % (
-                        port, API_KEY.encode(), API_PASSWORD.encode())
+                        port, API_KEY.encode('utf8'),
+                        API_PASSWORD.encode('utf8'))
                 os.write(fd, netrc)
             finally:
                 os.close(fd)
@@ -359,7 +360,7 @@ def respond(self, request):
         self.send_response(200)
         self.send_header("Content-Type", "application/json")
         self.end_headers()
-        self.wfile.write(b"%s" % (json.dumps({'foo': 'bar'}).encode(), ))
+        self.wfile.write(json.dumps({'foo': 'bar'}).encode('utf8'))
 
 
 class JSONTestCase(object):

From 5080b5a6ac3adfac18f9800a7093d8f03a153d5f Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Mon, 5 Jun 2017 14:02:22 -0400
Subject: [PATCH 86/90] Encode string after constructing

---
 tests.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests.py b/tests.py
index 241a70a..23e5297 100644
--- a/tests.py
+++ b/tests.py
@@ -303,9 +303,9 @@ def test_netrc(self):
                     address, port = address
                 else:
                     port = self.server.server_port
-                netrc = b"machine 127.0.0.1:%i\n  login %s\n  password %s" % (
-                        port, API_KEY.encode('utf8'),
-                        API_PASSWORD.encode('utf8'))
+                netrc = 'machine 127.0.0.1:%i\n  login %s\n  password %s' % (
+                        port, API_KEY, API_PASSWORD)
+                netrc = netrc.encode('utf8')
                 os.write(fd, netrc)
             finally:
                 os.close(fd)

From 77eab288ae870f35922a44d0f22e479c62dad1dc Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Mon, 5 Jun 2017 14:48:14 -0400
Subject: [PATCH 87/90] Use correct byte count function

---
 test/test_smartfile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index f59fdd3..2e19ff6 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -44,7 +44,7 @@ def upload(self):
         f = StringIO(file_contents)
         f.seek(0)
         self.api.upload(TESTFN, f)
-        self.assertEquals(self.get_data()['size'], f.len)
+        self.assertEquals(self.get_data()['size'], f.tell())
 
     def download(self):
         self.api.download(TESTFN)

From e12c5b932df1c180c763a6bf1630b22928d88527 Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Tue, 6 Jun 2017 13:44:53 -0400
Subject: [PATCH 88/90] Allow the user to control the download

Allows the caller of the 'download' method to get the response object
back to download the item as needed. This allows the caller to
iteratively download in whatever chunk size is desired, to download to
whatever location is desired, etc.
---
 smartfile/__init__.py  | 16 ++++++++++++----
 test/test_smartfile.py |  3 +++
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 44f3b3d..958e9fa 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -67,7 +67,10 @@ def _do_request(self, request, url, **kwargs):
                 return response.text
         else:
             # This might be a file, so return it.
-            return response.raw
+            if kwargs.get('params', {}).get('raw', True):
+                return response.raw
+            else:
+                return response
 
     def _request(self, method, endpoint, id=None, **kwargs):
         "Handles retrying failed requests and error handling."
@@ -145,14 +148,19 @@ def upload(self, filename, fileobj):
         arg = (filename, fileobj)
         return self.post('/path/data/', file=arg)
 
-    def download(self, file_to_be_downloaded):
+    def download(self, file_to_be_downloaded, perform_download=True):
         """ file_to_be_downloaded is a file-like object that has already
         been uploaded, you cannot download folders """
+        response = self.get(
+            '/path/data/', file_to_be_downloaded, raw=False)
+        if not perform_download:
+            # The caller can decide how to process the download of the data
+            return response
+
         # download uses shutil.copyfileobj to download, which copies
         # the data in chunks
         o = open(file_to_be_downloaded, 'wb')
-        return shutil.copyfileobj(self.get('/path/data/',
-                                  file_to_be_downloaded), o)
+        return shutil.copyfileobj(response.raw, o)
 
     def move(self, src_path, dst_path):
         # check destination folder for / at end
diff --git a/test/test_smartfile.py b/test/test_smartfile.py
index 2e19ff6..247de6c 100644
--- a/test/test_smartfile.py
+++ b/test/test_smartfile.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 
 import os
+import requests
 import unittest
 
 try:
@@ -47,6 +48,8 @@ def upload(self):
         self.assertEquals(self.get_data()['size'], f.tell())
 
     def download(self):
+        response = self.api.download(TESTFN, False)
+        self.assertTrue(isinstance(response, requests.Response))
         self.api.download(TESTFN)
         self.assertEquals(self.get_data()['size'], os.path.getsize(TESTFN))
 

From 4ef06893e858378e65f0ec7fb2b969919c9acc0b Mon Sep 17 00:00:00 2001
From: Clifton Barnes <cbarnes@smartfile.com>
Date: Wed, 7 Jun 2017 09:11:19 -0400
Subject: [PATCH 89/90] Fix pypi badge

---
 README.rst | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/README.rst b/README.rst
index d5574e3..0f4c0ef 100644
--- a/README.rst
+++ b/README.rst
@@ -12,14 +12,10 @@ uses and contributes to Open Source software.
     :target: https://coveralls.io/r/smartfile/client-python
     :alt: Code Coverage
 
-.. image:: https://pypip.in/v/smartfile/badge.png
-    :target: https://crate.io/packages/smartfile/
+.. image:: https://badge.fury.io/py/smartfile.svg
+    :target: https://badge.fury.io/py/smartfile
     :alt: Latest PyPI version
 
-.. image:: https://pypip.in/d/smartfile/badge.png
-    :target: https://crate.io/packages/smartfile/
-    :alt: Number of PyPI downloads
-
 Summary
 ------------
 

From 89a816a4fc30b562002b8b5c14189ad6b4afb52c Mon Sep 17 00:00:00 2001
From: Chaps  SD <drumchaps@gmail.com>
Date: Fri, 13 Apr 2018 10:44:24 -0400
Subject: [PATCH 90/90] Adds the download_to_path  keyword argument to the
 download method.

Edits smartfile/__init__.py
 - Adds a download_to_path keyword argument to explicitly specify a
 path to write the downloaded file contents , if not given the file
 will be written under the CWD + the file to download name.
---
 smartfile/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/smartfile/__init__.py b/smartfile/__init__.py
index 958e9fa..fe64074 100644
--- a/smartfile/__init__.py
+++ b/smartfile/__init__.py
@@ -148,7 +148,7 @@ def upload(self, filename, fileobj):
         arg = (filename, fileobj)
         return self.post('/path/data/', file=arg)
 
-    def download(self, file_to_be_downloaded, perform_download=True):
+    def download(self, file_to_be_downloaded, perform_download=True, download_to_path=None):
         """ file_to_be_downloaded is a file-like object that has already
         been uploaded, you cannot download folders """
         response = self.get(
@@ -156,10 +156,11 @@ def download(self, file_to_be_downloaded, perform_download=True):
         if not perform_download:
             # The caller can decide how to process the download of the data
             return response
-
+        if not download_to_path:
+            download_to_path = file_to_be_downloaded.split("/")[-1]
         # download uses shutil.copyfileobj to download, which copies
         # the data in chunks
-        o = open(file_to_be_downloaded, 'wb')
+        o = open(download_to_path, 'wb')
         return shutil.copyfileobj(response.raw, o)
 
     def move(self, src_path, dst_path):