diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8cdd1c7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,63 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +tests/media/ +coverage_html_report/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IDEs +.idea/ +*.sw[po] diff --git a/.travis.yml b/.travis.yml index 20c14c1..faf3129 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,9 @@ python: - "2.6" - "2.7" - "3.2" + - "3.3" + - "3.4" + - "pypy" before_install: - sudo apt-get install -qq librsync1 install: @@ -16,3 +19,5 @@ script: - make test after_success: - coveralls +notifications: + slack: smartfile:tbDIPzVJIPBpSz29kQw6b8RQ diff --git a/LICENSE b/LICENSE index 9877351..135d108 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ Copyright (c) 2013, Ben Timby +Copyright (c) 2015, SmartFile Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.rst b/README.rst index f5700cd..a40380e 100644 --- a/README.rst +++ b/README.rst @@ -41,13 +41,13 @@ two local files. import librsync # The destination file. - dst = file('Resume-v1.0.pdf', 'rb') + dst = open('Resume-v1.0.pdf', 'rb') # The source file. - src = file('Resume-v1.2.pdf', 'rb') + src = open('Resume-v1.2.pdf', 'rb') # Where we will write the synchronized copy. - synced = file('Resume-latest.pdf', 'wb') + synced = open('Resume-latest.pdf', 'wb') # Step 1: prepare signature of the destination file signature = librsync.signature(dst) diff --git a/librsync/__init__.py b/librsync/__init__.py index f9d0348..cd367f5 100644 --- a/librsync/__init__.py +++ b/librsync/__init__.py @@ -39,8 +39,9 @@ RS_DEFAULT_BLOCK_LEN = 2048 -# DEFINES FROM librsync.h: -#------------------------- +############################# +# DEFINES FROM librsync.h # +############################# # librsync.h: rs_buffers_s class Buffer(ctypes.Structure): @@ -94,14 +95,14 @@ class Buffer(ctypes.Structure): _librsync.rs_job_free.argtypes = (ctypes.c_void_p, ) # A function declaration for our read callback. -patch_callback = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_int, ctypes.c_size_t, - ctypes.POINTER(Buffer)) +patch_callback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_longlong, + ctypes.c_size_t, ctypes.POINTER(Buffer)) class LibrsyncError(Exception): - def __init__(self, result): - super(LibrsyncError, self).__init__( - _librsync.rs_strerror(ctypes.c_int(result))) + def __init__(self, r): + super(LibrsyncError, self).__init__(_librsync.rs_strerror( + ctypes.c_int(r))) def seekable(f): @@ -132,13 +133,13 @@ def _execute(job, f, o=None): # Set up our buffer for output. buff.next_out = ctypes.cast(out, ctypes.c_char_p) buff.avail_out = ctypes.c_size_t(RS_JOB_BLOCKSIZE) - result = _librsync.rs_job_iter(job, ctypes.byref(buff)) + r = _librsync.rs_job_iter(job, ctypes.byref(buff)) if o: o.write(out.raw[:RS_JOB_BLOCKSIZE - buff.avail_out]) - if result == RS_DONE: + if r == RS_DONE: break - elif result != RS_BLOCKED: - raise LibrsyncError(result) + elif r != RS_BLOCKED: + raise LibrsyncError(r) if buff.avail_in > 0: # There is data left in the input buffer, librsync did not consume # all of it. Rewind the file a bit so we include that data in our @@ -165,7 +166,7 @@ def signature(f, s=None, block_size=RS_DEFAULT_BLOCK_LEN): optional `block_size` parameter. """ if s is None: - s = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb') + s = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+') job = _librsync.rs_sig_begin(block_size, RS_DEFAULT_STRONG_LEN) try: _execute(job, f, s) @@ -183,15 +184,17 @@ def delta(f, s, d=None): objects. """ if d is None: - d = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb') + d = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+') sig = ctypes.c_void_p() - job = _librsync.rs_loadsig_begin(ctypes.byref(sig)) try: - _execute(job, s) - finally: - _librsync.rs_job_free(job) - try: - _librsync.rs_build_hash_table(sig) + job = _librsync.rs_loadsig_begin(ctypes.byref(sig)) + try: + _execute(job, s) + finally: + _librsync.rs_job_free(job) + r = _librsync.rs_build_hash_table(sig) + if r != RS_DONE: + raise LibrsyncError(r) job = _librsync.rs_delta_begin(sig) try: _execute(job, f, d) @@ -211,14 +214,18 @@ def patch(f, d, o=None): required to be seekable. """ if o is None: - o = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb') + o = tempfile.SpooledTemporaryFile(max_size=MAX_SPOOL, mode='wb+') @patch_callback - def read_cb(pos, length, buff): + def read_cb(opaque, pos, length, buff): f.seek(pos) - block = f.read(length) - buff.next_in = ctypes.c_char_p(block) - buff.avail_in = ctypes.c_size_t(len(block)) + size_p = ctypes.cast(length, ctypes.POINTER(ctypes.c_size_t)).contents + size = size_p.value + block = f.read(size) + size_p.value = len(block) + buff_p = ctypes.cast(buff, ctypes.POINTER(ctypes.c_char_p)).contents + buff_p.value = block + return RS_DONE job = _librsync.rs_patch_begin(read_cb, None) try: diff --git a/requirements.txt b/requirements.txt index 9d64331..4490864 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -coverage +coverage<4 coveralls diff --git a/tests.py b/tests.py index dd31a40..e5a87eb 100644 --- a/tests.py +++ b/tests.py @@ -4,9 +4,9 @@ import librsync try: - from StringIO import StringIO + from BytesIO import BytesIO except ImportError: - from io import BytesIO as StringIO + from io import BytesIO class TraceLevelTestCase(unittest.TestCase): @@ -19,13 +19,13 @@ def test_set_invalid(self): class SingleFileTestCase(unittest.TestCase): def setUp(self): - self.rand = StringIO(os.urandom(1024**2)) + self.rand = BytesIO(os.urandom(1024**2)) class DoubleFileTestCase(unittest.TestCase): def setUp(self): - self.rand1 = StringIO(os.urandom(1024**2)) - self.rand2 = StringIO(os.urandom(1024**2)) + self.rand1 = BytesIO(os.urandom(1024**2)) + self.rand2 = BytesIO(os.urandom(1024**2)) class SignatureTestCase(SingleFileTestCase): @@ -62,5 +62,32 @@ def test_failure(self): self.rand2) +class BigPatchTestCase(PatchTestCase): + def setUp(self): + "Use large enough test files to cause temp files to hit disk." + self.rand1 = BytesIO(os.urandom(1024**2*5)) + self.rand2 = BytesIO(os.urandom(1024**2*5)) + + +class Issue3TestCase(PatchTestCase): + def setUp(self): + "Use test data provided in issue #3." + self.rand1 = BytesIO(b'Text.') + self.rand2 = BytesIO(b'New text.\nText.') + + +class SimpleStringTestCase(unittest.TestCase): + def setUp(self): + self.src = b'FF' + self.dst = b'FF123FF' + + def test_string_patch(self): + src_sig = librsync.signature(BytesIO(self.src)) + delta = librsync.delta(BytesIO(self.dst), src_sig).read() + out = librsync.patch(BytesIO(self.src), BytesIO(delta)) + + self.assertEqual(self.dst, out.read()) + + if __name__ == '__main__': unittest.main()