From 23dbf700da8e296aa2f5b2726a57d849100b16e7 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 30 Apr 2016 16:57:20 +0900 Subject: [PATCH 001/109] add support for building universal wheel Signed-off-by: Vanya A. Sergeev --- setup.cfg | 2 ++ setup.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..2be6836 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal = True diff --git a/setup.py b/setup.py index fa314b0..6dbf84e 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,7 @@ -from distutils.core import setup +try: + from setuptools import setup +except ImportError: + from distutils.core import setup setup( name='u-msgpack-python', From df59af65f2ab447ba45dd3fe0815196254c48068 Mon Sep 17 00:00:00 2001 From: Fairiz 'Fi' Azizi Date: Fri, 9 Sep 2016 00:34:51 -0700 Subject: [PATCH 002/109] add support for bytearray type in loads/unpackb Some libraries (such as the haigha amqp library), ends up representing the packed byte sequence as a bytearray. This patch will allow the loads/unpackb method to accept this type as well as the str type. Signed-off-by: Vanya A. Sergeev --- umsgpack.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index 76a157b..e74d97d 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -713,14 +713,14 @@ def _unpackb2(s): Deserialize MessagePack bytes into a Python object. Args: - s: a 'str' containing serialized MessagePack bytes + s: a 'str' or 'bytearray' containing serialized MessagePack bytes Returns: A Python object. Raises: TypeError: - Packed data is not type 'str'. + Packed data type is neither 'str' nor 'bytearray'. InsufficientDataException(UnpackException): Insufficient data to unpack the encoded object. InvalidStringException(UnpackException): @@ -736,10 +736,12 @@ def _unpackb2(s): Example: >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') {u'compact': True, u'schema': 0} + >>> umsgpack.unpackb(bytearray(b'\x82\xa7compact\xc3\xa6schema\x00')) + {u'compact': True, u'schema': 0} >>> """ - if not isinstance(s, str): - raise TypeError("packed data is not type 'str'") + if not isinstance(s, (str, bytearray)): + raise TypeError("packed data must be type 'str' or 'bytearray'") return _unpack(io.BytesIO(s)) # For Python 3, expects a bytes object From 214ec38c0229ed0e2d1277804c4735dc0ed7a932 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 01:11:13 -0700 Subject: [PATCH 003/109] use OrderedDict in map unit test vectors for packing determinism fixes #13. --- test_umsgpack.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index 977b24b..cd80fca 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -8,11 +8,13 @@ # $ pypy3 test_umsgpack.py # -import umsgpack import sys import struct import unittest import io +from collections import OrderedDict + +import umsgpack single_test_vectors = [ # None @@ -120,19 +122,19 @@ # 32-bit Array [ "32-bit array", [ 0x05 ]*65536, b"\xdd\x00\x01\x00\x00" + b"\x05"*65536 ], # Fix Map - [ "fix map", { 1: True, 2: u"abc", 3: b"\x80" }, b"\x83\x01\xc3\x02\xa3\x61\x62\x63\x03\xc4\x01\x80" ], + [ "fix map", OrderedDict([(1, True), (2, u"abc"), (3, b"\x80")]), b"\x83\x01\xc3\x02\xa3\x61\x62\x63\x03\xc4\x01\x80" ], [ "fix map", { u"abc" : 5 }, b"\x81\xa3\x61\x62\x63\x05" ], [ "fix map", { b"\x80" : 0xffff }, b"\x81\xc4\x01\x80\xcd\xff\xff" ], [ "fix map", { True : None }, b"\x81\xc3\xc0" ], # 16-bit Map - [ "16-bit map", dict([(k, 0x05) for k in range(16)]), b"\xde\x00\x10" + b"".join( [ struct.pack("B", i) + b"\x05" for i in range(16) ] ) ], - [ "16-bit map", dict([(k, 0x05) for k in range(6000)]), b"\xde\x17\x70" + b"".join([ struct.pack("B", i) + b"\x05" for i in range(128)]) + b"".join([ b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + b"".join([ b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)]) ], + [ "16-bit map", OrderedDict([(k, 0x05) for k in range(16)]), b"\xde\x00\x10" + b"".join( [ struct.pack("B", i) + b"\x05" for i in range(16)])], + [ "16-bit map", OrderedDict([(k, 0x05) for k in range(6000)]), b"\xde\x17\x70" + b"".join([ struct.pack("B", i) + b"\x05" for i in range(128)]) + b"".join([ b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + b"".join([ b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)]) ], # Complex Array - [ "complex array", [ True, 0x01, umsgpack.Ext(0x03, b"foo"), 0xff, { 1: False, 2: u"abc" }, b"\x80", [ 1, 2, 3], u"abc" ], b"\x98\xc3\x01\xc7\x03\x03\x66\x6f\x6f\xcc\xff\x82\x01\xc2\x02\xa3\x61\x62\x63\xc4\x01\x80\x93\x01\x02\x03\xa3\x61\x62\x63" ], + [ "complex array", [ True, 0x01, umsgpack.Ext(0x03, b"foo"), 0xff, OrderedDict([(1, False), (2, u"abc")]), b"\x80", [1, 2, 3], u"abc" ], b"\x98\xc3\x01\xc7\x03\x03\x66\x6f\x6f\xcc\xff\x82\x01\xc2\x02\xa3\x61\x62\x63\xc4\x01\x80\x93\x01\x02\x03\xa3\x61\x62\x63" ], # Complex Map - [ "complex map", { 1 : [{1: 2, 3: 4}, {}], 2: 1, 3: [False, u"def"], 4: {0x100000000: u"a", 0xffffffff: u"b"}}, b"\x84\x01\x92\x82\x01\x02\x03\x04\x80\x02\x01\x03\x92\xc2\xa3\x64\x65\x66\x04\x82\xcf\x00\x00\x00\x01\x00\x00\x00\x00\xa1\x61\xce\xff\xff\xff\xff\xa1\x62" ], + [ "complex map", OrderedDict([(1, [OrderedDict([(1, 2), (3, 4)]), {}]), (2, 1), (3, [False, u"def"]), (4, OrderedDict([(0x100000000, u"a"), (0xffffffff, u"b")]))]), b"\x84\x01\x92\x82\x01\x02\x03\x04\x80\x02\x01\x03\x92\xc2\xa3\x64\x65\x66\x04\x82\xcf\x00\x00\x00\x01\x00\x00\x00\x00\xa1\x61\xce\xff\xff\xff\xff\xa1\x62" ], # Map with Tuple Keys - [ "map with tuple keys", {(u"foo", False, 3) : True, (3e6, -5): u"def"}, b"\x82\x92\xcb\x41\x46\xe3\x60\x00\x00\x00\x00\xfb\xa3\x64\x65\x66\x93\xa3\x66\x6f\x6f\xc2\x03\xc3" ], + [ "map with tuple keys", OrderedDict([((u"foo", False, 3), True), ((3e6, -5), u"def")]), b"\x82\x93\xa3\x66\x6f\x6f\xc2\x03\xc3\x92\xcb\x41\x46\xe3\x60\x00\x00\x00\x00\xfb\xa3\x64\x65\x66" ], # Map with Complex Tuple Keys [ "map with complex tuple keys", {(u"foo", (1,2,3), 3) : -5}, b"\x81\x93\xa3\x66\x6f\x6f\x93\x01\x02\x03\x03\xfb" ] ] From 6859c81b1e10544de40493891afd47cc1f789ad3 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 01:18:56 -0700 Subject: [PATCH 004/109] simplify test vector synthesis and clean up unit tests --- test_umsgpack.py | 71 ++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index cd80fca..aecda75 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -43,9 +43,9 @@ [ "32-bit uint", 0x200000, b"\xce\x00\x20\x00\x00" ], [ "32-bit uint", 0xffffffff, b"\xce\xff\xff\xff\xff" ], # 64-bit uint - [ "64-bit uint", 0x100000000, b"\xcf" + b"\x00\x00\x00\x01" + b"\x00\x00\x00\x00" ], - [ "64-bit uint", 0x200000000000, b"\xcf" + b"\x00\x00\x20\x00" + b"\x00\x00\x00\x00" ], - [ "64-bit uint", 0xffffffffffffffff, b"\xcf" + b"\xff\xff\xff\xff" + b"\xff\xff\xff\xff" ], + [ "64-bit uint", 0x100000000, b"\xcf\x00\x00\x00\x01\x00\x00\x00\x00" ], + [ "64-bit uint", 0x200000000000, b"\xcf\x00\x00\x20\x00\x00\x00\x00\x00" ], + [ "64-bit uint", 0xffffffffffffffff, b"\xcf\xff\xff\xff\xff\xff\xff\xff\xff" ], # 8-bit int [ "8-bit int", -33, b"\xd0\xdf" ], [ "8-bit int", -100, b"\xd0\x9c" ], @@ -59,38 +59,38 @@ [ "32-bit int", -1000000000, b"\xd2\xc4\x65\x36\x00" ], [ "32-bit int", -2147483648, b"\xd2\x80\x00\x00\x00" ], # 64-bit int - [ "64-bit int", -2147483649, b"\xd3" + b"\xff\xff\xff\xff" + b"\x7f\xff\xff\xff" ], - [ "64-bit int", -1000000000000000002, b"\xd3" + b"\xf2\x1f\x49\x4c" + b"\x58\x9b\xff\xfe" ], - [ "64-bit int", -9223372036854775808, b"\xd3" + b"\x80\x00\x00\x00" + b"\x00\x00\x00\x00" ], + [ "64-bit int", -2147483649, b"\xd3\xff\xff\xff\xff\x7f\xff\xff\xff" ], + [ "64-bit int", -1000000000000000002, b"\xd3\xf2\x1f\x49\x4c\x58\x9b\xff\xfe" ], + [ "64-bit int", -9223372036854775808, b"\xd3\x80\x00\x00\x00\x00\x00\x00\x00" ], # 64-bit float - [ "64-bit float", 0.0, b"\xcb" + b"\x00\x00\x00\x00" + b"\x00\x00\x00\x00" ], - [ "64-bit float", 2.5, b"\xcb" + b"\x40\x04\x00\x00" + b"\x00\x00\x00\x00" ], - [ "64-bit float", float(10**35), b"\xcb" + b"\x47\x33\x42\x61" + b"\x72\xc7\x4d\x82" ], + [ "64-bit float", 0.0, b"\xcb\x00\x00\x00\x00\x00\x00\x00\x00" ], + [ "64-bit float", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00" ], + [ "64-bit float", float(10**35), b"\xcb\x47\x33\x42\x61\x72\xc7\x4d\x82" ], # Fixstr String [ "fix string", u"", b"\xa0" ], [ "fix string", u"a", b"\xa1\x61" ], [ "fix string", u"abc", b"\xa3\x61\x62\x63" ], - [ "fix string", u"a" * 31, b"\xbf" + b"\x61"*31 ], + [ "fix string", u"a"*31, b"\xbf" + b"\x61"*31 ], # 8-bit String - [ "8-bit string", u"b" * 32, b"\xd9\x20" + b"b" * 32 ], - [ "8-bit string", u"c" * 100, b"\xd9\x64" + b"c" * 100 ], - [ "8-bit string", u"d" * 255, b"\xd9\xff" + b"d" * 255 ], + [ "8-bit string", u"b"*32, b"\xd9\x20" + b"b"*32 ], + [ "8-bit string", u"c"*100, b"\xd9\x64" + b"c"*100 ], + [ "8-bit string", u"d"*255, b"\xd9\xff" + b"d"*255 ], # 16-bit String - [ "16-bit string", u"b" * 256, b"\xda\x01\x00" + b"b" * 256 ], - [ "16-bit string", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535 ], + [ "16-bit string", u"b"*256, b"\xda\x01\x00" + b"b"*256 ], + [ "16-bit string", u"c"*65535, b"\xda\xff\xff" + b"c"*65535 ], # 32-bit String - [ "32-bit string", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536 ], + [ "32-bit string", u"b"*65536, b"\xdb\x00\x01\x00\x00" + b"b"*65536 ], # Wide character String [ "wide char string", u"Allagbé", b"\xa8Allagb\xc3\xa9" ], [ "wide char string", u"По оживлённым берегам", b"\xd9\x28\xd0\x9f\xd0\xbe\x20\xd0\xbe\xd0\xb6\xd0\xb8\xd0\xb2\xd0\xbb\xd1\x91\xd0\xbd\xd0\xbd\xd1\x8b\xd0\xbc\x20\xd0\xb1\xd0\xb5\xd1\x80\xd0\xb5\xd0\xb3\xd0\xb0\xd0\xbc" ], # 8-bit Binary - [ "8-bit binary", b"\x80" * 1, b"\xc4\x01" + b"\x80" * 1 ], - [ "8-bit binary", b"\x80" * 32, b"\xc4\x20" + b"\x80" * 32 ], - [ "8-bit binary", b"\x80" * 255, b"\xc4\xff" + b"\x80" * 255 ], + [ "8-bit binary", b"\x80"*1, b"\xc4\x01" + b"\x80"*1 ], + [ "8-bit binary", b"\x80"*32, b"\xc4\x20" + b"\x80"*32 ], + [ "8-bit binary", b"\x80"*255, b"\xc4\xff" + b"\x80"*255 ], # 16-bit Binary - [ "16-bit binary", b"\x80" * 256, b"\xc5\x01\x00" + b"\x80" * 256 ], + [ "16-bit binary", b"\x80"*256, b"\xc5\x01\x00" + b"\x80"*256 ], # 32-bit Binary - [ "32-bit binary", b"\x80" * 65536, b"\xc6\x00\x01\x00\x00" + b"\x80" * 65536 ], + [ "32-bit binary", b"\x80"*65536, b"\xc6\x00\x01\x00\x00" + b"\x80"*65536 ], # Fixext 1 [ "fixext 1", umsgpack.Ext(0x05, b"\x80"*1), b"\xd4\x05" + b"\x80"*1 ], # Fixext 2 @@ -127,8 +127,8 @@ [ "fix map", { b"\x80" : 0xffff }, b"\x81\xc4\x01\x80\xcd\xff\xff" ], [ "fix map", { True : None }, b"\x81\xc3\xc0" ], # 16-bit Map - [ "16-bit map", OrderedDict([(k, 0x05) for k in range(16)]), b"\xde\x00\x10" + b"".join( [ struct.pack("B", i) + b"\x05" for i in range(16)])], - [ "16-bit map", OrderedDict([(k, 0x05) for k in range(6000)]), b"\xde\x17\x70" + b"".join([ struct.pack("B", i) + b"\x05" for i in range(128)]) + b"".join([ b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + b"".join([ b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)]) ], + [ "16-bit map", OrderedDict([(k, 0x05) for k in range(16)]), b"\xde\x00\x10" + b"".join([struct.pack("B", i) + b"\x05" for i in range(16)])], + [ "16-bit map", OrderedDict([(k, 0x05) for k in range(6000)]), b"\xde\x17\x70" + b"".join([struct.pack("B", i) + b"\x05" for i in range(128)]) + b"".join([b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + b"".join([b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)]) ], # Complex Array [ "complex array", [ True, 0x01, umsgpack.Ext(0x03, b"foo"), 0xff, OrderedDict([(1, False), (2, u"abc")]), b"\x80", [1, 2, 3], u"abc" ], b"\x98\xc3\x01\xc7\x03\x03\x66\x6f\x6f\xcc\xff\x82\x01\xc2\x02\xa3\x61\x62\x63\xc4\x01\x80\x93\x01\x02\x03\xa3\x61\x62\x63" ], # Complex Map @@ -207,18 +207,18 @@ [ "fix raw", u"a", b"\xa1\x61" ], [ "fix raw", b"abc", b"\xa3\x61\x62\x63" ], [ "fix raw", u"abc", b"\xa3\x61\x62\x63" ], - [ "fix raw", b"a" * 31, b"\xbf" + b"\x61"*31 ], - [ "fix raw", u"a" * 31, b"\xbf" + b"\x61"*31 ], + [ "fix raw", b"a"*31, b"\xbf" + b"\x61"*31 ], + [ "fix raw", u"a"*31, b"\xbf" + b"\x61"*31 ], # 16-bit Raw - [ "16-bit raw", u"b" * 32, b"\xda\x00\x20" + b"b" * 32 ], - [ "16-bit raw", b"b" * 32, b"\xda\x00\x20" + b"b" * 32 ], - [ "16-bit raw", u"b" * 256, b"\xda\x01\x00" + b"b" * 256 ], - [ "16-bit raw", b"b" * 256, b"\xda\x01\x00" + b"b" * 256 ], - [ "16-bit raw", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535 ], - [ "16-bit raw", b"c" * 65535, b"\xda\xff\xff" + b"c" * 65535 ], + [ "16-bit raw", u"b"*32, b"\xda\x00\x20" + b"b"*32 ], + [ "16-bit raw", b"b"*32, b"\xda\x00\x20" + b"b"*32 ], + [ "16-bit raw", u"b"*256, b"\xda\x01\x00" + b"b"*256 ], + [ "16-bit raw", b"b"*256, b"\xda\x01\x00" + b"b"*256 ], + [ "16-bit raw", u"c"*65535, b"\xda\xff\xff" + b"c"*65535 ], + [ "16-bit raw", b"c"*65535, b"\xda\xff\xff" + b"c"*65535 ], # 32-bit Raw - [ "32-bit raw", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536 ], - [ "32-bit raw", b"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536 ], + [ "32-bit raw", u"b"*65536, b"\xdb\x00\x01\x00\x00" + b"b"*65536 ], + [ "32-bit raw", b"b"*65536, b"\xdb\x00\x01\x00\x00" + b"b"*65536 ], ] # These are the only global variables that should be exported by umsgpack @@ -344,14 +344,14 @@ def test_ext_exceptions(self): def test_streaming_writer(self): # Try first composite test vector - (name, obj, data) = composite_test_vectors[0] + (_, obj, data) = composite_test_vectors[0] writer = io.BytesIO() umsgpack.pack(obj, writer) self.assertTrue(writer.getvalue(), data) def test_streaming_reader(self): # Try first composite test vector - (name, obj, data) = composite_test_vectors[0] + (_, obj, data) = composite_test_vectors[0] reader = io.BytesIO(data) self.assertEqual(umsgpack.unpack(reader), obj) @@ -367,4 +367,3 @@ def test_namespacing(self): if __name__ == '__main__': unittest.main() - From 201ea17f4da5b990284f123ebf93030e3973c204 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 02:05:19 -0700 Subject: [PATCH 005/109] add support for bytearray type in loads/unpackb for python 3 --- umsgpack.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index e74d97d..b9bc7fb 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -736,8 +736,6 @@ def _unpackb2(s): Example: >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') {u'compact': True, u'schema': 0} - >>> umsgpack.unpackb(bytearray(b'\x82\xa7compact\xc3\xa6schema\x00')) - {u'compact': True, u'schema': 0} >>> """ if not isinstance(s, (str, bytearray)): @@ -750,14 +748,14 @@ def _unpackb3(s): Deserialize MessagePack bytes into a Python object. Args: - s: a 'bytes' containing serialized MessagePack bytes + s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes Returns: A Python object. Raises: TypeError: - Packed data is not type 'bytes'. + Packed data type is neither 'bytes' nor 'bytearray'. InsufficientDataException(UnpackException): Insufficient data to unpack the encoded object. InvalidStringException(UnpackException): @@ -775,8 +773,8 @@ def _unpackb3(s): {'compact': True, 'schema': 0} >>> """ - if not isinstance(s, bytes): - raise TypeError("packed data is not type 'bytes'") + if not isinstance(s, (bytes, bytearray)): + raise TypeError("packed data must be type 'bytes' or 'bytearray'") return _unpack(io.BytesIO(s)) ################################################################################ From 02004fbaa41d3dace65a7d49ea9d92c9b797e5fa Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 02:58:35 -0700 Subject: [PATCH 006/109] add support for unpacking maps into OrderedDict resolves #12. --- test_umsgpack.py | 17 +++++++++++++ umsgpack.py | 65 +++++++++++++++++++++++++++++------------------- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index aecda75..81ada20 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -332,6 +332,23 @@ def test_unpack_compatibility(self): umsgpack.compatibility = False + def test_unpack_ordered_dict(self): + # Use last composite test vector (a map) + (_, obj, data) = composite_test_vectors[-1] + + # Unpack with default options (unordered dict) + unpacked = umsgpack.unpackb(data) + self.assertTrue(isinstance(unpacked, dict)) + + # Unpack with unordered dict + unpacked = umsgpack.unpackb(data, use_ordered_dict=False) + self.assertTrue(isinstance(unpacked, dict)) + + # Unpack with ordered dict + unpacked = umsgpack.unpackb(data, use_ordered_dict=True) + self.assertTrue(isinstance(unpacked, OrderedDict)) + self.assertEqual(unpacked, obj) + def test_ext_exceptions(self): with self.assertRaises(TypeError): _ = umsgpack.Ext(-1, b"") diff --git a/umsgpack.py b/umsgpack.py index b9bc7fb..b847ed5 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -487,7 +487,7 @@ def _read_except(fp, n): raise InsufficientDataException() return data -def _unpack_integer(code, fp): +def _unpack_integer(code, fp, options): if (ord(code) & 0xe0) == 0xe0: return struct.unpack("b", code)[0] elif code == b'\xd0': @@ -510,31 +510,31 @@ def _unpack_integer(code, fp): return struct.unpack(">Q", _read_except(fp, 8))[0] raise Exception("logic error, not int: 0x%02x" % ord(code)) -def _unpack_reserved(code, fp): +def _unpack_reserved(code, fp, options): if code == b'\xc1': raise ReservedCodeException("encountered reserved code: 0x%02x" % ord(code)) raise Exception("logic error, not reserved code: 0x%02x" % ord(code)) -def _unpack_nil(code, fp): +def _unpack_nil(code, fp, options): if code == b'\xc0': return None raise Exception("logic error, not nil: 0x%02x" % ord(code)) -def _unpack_boolean(code, fp): +def _unpack_boolean(code, fp, options): if code == b'\xc2': return False elif code == b'\xc3': return True raise Exception("logic error, not boolean: 0x%02x" % ord(code)) -def _unpack_float(code, fp): +def _unpack_float(code, fp, options): if code == b'\xca': return struct.unpack(">f", _read_except(fp, 4))[0] elif code == b'\xcb': return struct.unpack(">d", _read_except(fp, 8))[0] raise Exception("logic error, not float: 0x%02x" % ord(code)) -def _unpack_string(code, fp): +def _unpack_string(code, fp, options): if (ord(code) & 0xe0) == 0xa0: length = ord(code) & ~0xe0 elif code == b'\xd9': @@ -556,7 +556,7 @@ def _unpack_string(code, fp): except UnicodeDecodeError: raise InvalidStringException("unpacked string is not utf-8") -def _unpack_binary(code, fp): +def _unpack_binary(code, fp, options): if code == b'\xc4': length = struct.unpack("B", _read_except(fp, 1))[0] elif code == b'\xc5': @@ -568,7 +568,7 @@ def _unpack_binary(code, fp): return _read_except(fp, length) -def _unpack_ext(code, fp): +def _unpack_ext(code, fp, options): if code == b'\xd4': length = 1 elif code == b'\xd5': @@ -590,7 +590,7 @@ def _unpack_ext(code, fp): return Ext(ord(_read_except(fp, 1)), _read_except(fp, length)) -def _unpack_array(code, fp): +def _unpack_array(code, fp, options): if (ord(code) & 0xf0) == 0x90: length = (ord(code) & ~0xf0) elif code == b'\xdc': @@ -600,14 +600,14 @@ def _unpack_array(code, fp): else: raise Exception("logic error, not array: 0x%02x" % ord(code)) - return [_unpack(fp) for i in xrange(length)] + return [_unpack(fp, options) for i in xrange(length)] def _deep_list_to_tuple(obj): if isinstance(obj, list): return tuple([_deep_list_to_tuple(e) for e in obj]) return obj -def _unpack_map(code, fp): +def _unpack_map(code, fp, options): if (ord(code) & 0xf0) == 0x80: length = (ord(code) & ~0xf0) elif code == b'\xde': @@ -617,10 +617,10 @@ def _unpack_map(code, fp): else: raise Exception("logic error, not map: 0x%02x" % ord(code)) - d = {} - for i in xrange(length): + d = {} if not options.get('use_ordered_dict') else collections.OrderedDict() + for _ in xrange(length): # Unpack key - k = _unpack(fp) + k = _unpack(fp, options) if isinstance(k, list): # Attempt to convert list into a hashable tuple @@ -631,7 +631,7 @@ def _unpack_map(code, fp): raise DuplicateKeyException("encountered duplicate key: %s, %s" % (str(k), str(type(k)))) # Unpack value - v = _unpack(fp) + v = _unpack(fp, options) try: d[k] = v @@ -639,19 +639,23 @@ def _unpack_map(code, fp): raise UnhashableKeyException("encountered unhashable key: %s" % str(k)) return d -def _unpack(fp): +def _unpack(fp, options): code = _read_except(fp, 1) - return _unpack_dispatch_table[code](code, fp) + return _unpack_dispatch_table[code](code, fp, options) ######################################## -def _unpack2(fp): +def _unpack2(fp, **options): """ Deserialize MessagePack bytes into a Python object. Args: fp: a .read()-supporting file-like object + Kwargs: + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) + Returns: A Python object. @@ -674,15 +678,19 @@ def _unpack2(fp): {u'compact': True, u'schema': 0} >>> """ - return _unpack(fp) + return _unpack(fp, options) -def _unpack3(fp): +def _unpack3(fp, **options): """ Deserialize MessagePack bytes into a Python object. Args: fp: a .read()-supporting file-like object + Kwargs: + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) + Returns: A Python object. @@ -705,16 +713,20 @@ def _unpack3(fp): {'compact': True, 'schema': 0} >>> """ - return _unpack(fp) + return _unpack(fp, options) # For Python 2, expects a str object -def _unpackb2(s): +def _unpackb2(s, **options): """ Deserialize MessagePack bytes into a Python object. Args: s: a 'str' or 'bytearray' containing serialized MessagePack bytes + Kwargs: + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) + Returns: A Python object. @@ -740,16 +752,19 @@ def _unpackb2(s): """ if not isinstance(s, (str, bytearray)): raise TypeError("packed data must be type 'str' or 'bytearray'") - return _unpack(io.BytesIO(s)) + return _unpack(io.BytesIO(s), options) # For Python 3, expects a bytes object -def _unpackb3(s): +def _unpackb3(s, **options): """ Deserialize MessagePack bytes into a Python object. Args: s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes + Kwargs: + use_ordered_dict (bool): unpack maps into OrderedDict, instead of + unordered dict (default False) Returns: A Python object. @@ -775,7 +790,7 @@ def _unpackb3(s): """ if not isinstance(s, (bytes, bytearray)): raise TypeError("packed data must be type 'bytes' or 'bytearray'") - return _unpack(io.BytesIO(s)) + return _unpack(io.BytesIO(s), options) ################################################################################ ### Module Initialization From fe1e69574a78135b5aeef3ead52a5944ca4af7f3 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 02:59:55 -0700 Subject: [PATCH 007/109] add use_ordered_dict option usage to readme --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c7df1c..940a03b 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,21 @@ The streaming `pack()`/`dump()` and `unpack()`/`load()` functions allow packing >>> ``` -## Compatibility Mode +## Options + +### Ordered Dictionaries + +The unpacking functions provide a `use_ordered_dict` option to unpack MessagePack maps into the `collections.OrderedDict` type, rather than the unordered `dict` type, to preserve the order of deserialized MessagePack maps. + +``` python +>>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') +{'compact': True, 'schema': 0} +>>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00', use_ordered_dict=True) +OrderedDict([('compact', True), ('schema', 0)]) +>>> +``` + +### Compatibility Mode u-msgpack-python offers a compatibility mode for the [old specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md) to handle the old "raw" bytes msgpack type. When the compatibility mode is enabled, u-msgpack-python will serialize both unicode strings and bytes into the old "raw" msgpack type, and deserialize the "raw" msgpack type into bytes. To enable compatibility mode, simply set the `compatibility` boolean of the umsgpack module to `True`. From 329f04f9d237a227817ef683452418b522479c84 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 01:39:40 -0700 Subject: [PATCH 008/109] add readme filename to setup.cfg --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 2be6836..768d2bc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ +[metadata] +description-file = README.md + [bdist_wheel] universal = True From 83f270cb2ab4aa16d935710435cda6bc351cbd4b Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 01:40:00 -0700 Subject: [PATCH 009/109] add build/ and *.egg-info/ directories to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 7d481b4..bf98a32 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ MANIFEST +build/ dist/ +*.egg-info/ *.pyc *.swp From 618a50fcd4fda45148f121ed658dd0072fc7075f Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 01:49:08 -0700 Subject: [PATCH 010/109] add MANIFEST.in with license and unit test resolves #21. resolves #18. --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..6ffd061 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE +include test_umsgpack.py From 81911b3ad7d22308569f1ae0a1ebfb07b582bb71 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 01:53:07 -0700 Subject: [PATCH 011/109] update email address in setup.py --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6dbf84e..449bb62 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ version='2.1', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', - author_email='vsergeev at gmail', + author_email='v at sergeev.io', url='https://github.com/vsergeev/u-msgpack-python', py_modules=['umsgpack'], long_description="""u-msgpack-python is a lightweight `MessagePack `_ serializer and deserializer module written in pure Python, compatible with both Python 2 and Python 3, as well as CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest `MessagePack specification `_. In particular, it supports the new binary, UTF-8 string, and application-defined ext types. See https://github.com/vsergeev/u-msgpack-python for more information.""", @@ -24,5 +24,4 @@ ], license='MIT', keywords='msgpack serialization deserialization', - ) - +) From 10c50a8e258d92ee9277af978c777ad0048a446c Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 03:00:05 -0700 Subject: [PATCH 012/109] improve wording of readme and msgpack.org.md --- README.md | 8 ++++---- msgpack.org.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 940a03b..f30d47c 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ OrderedDict([('compact', True), ('schema', 0)]) ### Compatibility Mode -u-msgpack-python offers a compatibility mode for the [old specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md) to handle the old "raw" bytes msgpack type. When the compatibility mode is enabled, u-msgpack-python will serialize both unicode strings and bytes into the old "raw" msgpack type, and deserialize the "raw" msgpack type into bytes. To enable compatibility mode, simply set the `compatibility` boolean of the umsgpack module to `True`. +The compatibility mode supports the "raw" bytes MessagePack type from the [old specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md). When the module-wide `compatibility` option is enabled, both unicode strings and bytes will be serialized into the "raw" MessagePack type, and the "raw" MessagePack type will be deserialized into bytes. ``` python >>> umsgpack.compatibility = True @@ -238,7 +238,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * `UnhashableKeyException`: Unhashable key encountered during map unpacking. The packed map cannot be unpacked into a Python dictionary. - Python dictionaries only support keys that are instances of `collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a msgpack encoding, it cannot be unpacked into a valid Python dictionary. + Python dictionaries only support keys that are instances of `collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a MessagePack encoding, it cannot be unpacked into a valid Python dictionary. ``` python # Attempt to unpack { {} : False } @@ -250,7 +250,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * `DuplicateKeyException`: Duplicate key encountered during map unpacking. - Python dictionaries do not support duplicate keys, but msgpack maps may be encoded with duplicate keys. + Python dictionaries do not support duplicate keys, but MessagePack maps may be encoded with duplicate keys. ``` python # Attempt to unpack { 1: True, 1: False } @@ -275,7 +275,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a ## Testing -u-msgpack-python's included unit tests may be run with `test_umsgpack.py`, under your favorite interpreter. +The included unit tests may be run with `test_umsgpack.py`, under your favorite interpreter. ``` text $ python2 test_umsgpack.py diff --git a/msgpack.org.md b/msgpack.org.md index 7cd2c1b..7127588 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -101,7 +101,7 @@ available: ## More Information -See the [project page](https://github.com/vsergeev/u-msgpack-python) for more information on old specification compatibility mode, exceptions, behavior, and testing. +See the [project page](https://github.com/vsergeev/u-msgpack-python) for more information on options, exceptions, behavior, and testing. ## License From 1d1d86415139ae39379f71ba840fe5568a558149 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 03:24:40 -0700 Subject: [PATCH 013/109] update copyright years in license --- LICENSE | 2 +- umsgpack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index a0fbb2b..5b6471d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2013-2014 Ivan A. Sergeev + Copyright (c) 2013-2016 Ivan (Vanya) A. Sergeev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/umsgpack.py b/umsgpack.py index b847ed5..05855f0 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -10,7 +10,7 @@ # # MIT License # -# Copyright (c) 2013-2014 Ivan A. Sergeev +# Copyright (c) 2013-2016 Ivan (Vanya) A. Sergeev # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From 923b038da65b4650cc974872f5f349f47f66be09 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 03:11:33 -0700 Subject: [PATCH 014/109] add travis build configuration --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..5932b92 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: python +python: + - "2.7" + - "3.5" + - "pypy" + - "pypy3" +script: python test_umsgpack.py From a3a10717c3829b7bee10ac2c21f796f5bcd521c2 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 03:14:39 -0700 Subject: [PATCH 015/109] add build status, release, and license badges to readme and msgpack.org.md --- README.md | 2 +- msgpack.org.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f30d47c..d63f17b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# u-msgpack-python v2.1 +# u-msgpack-python [![Build Status](https://travis-ci.org/vsergeev/u-msgpack-python.svg?branch=master)](https://travis-ci.org/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, and application-defined ext types. diff --git a/msgpack.org.md b/msgpack.org.md index 7127588..5cf18b0 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -1,4 +1,4 @@ -# u-msgpack-python v2.1 +# u-msgpack-python [![Build Status](https://travis-ci.org/vsergeev/u-msgpack-python.svg?branch=master)](https://travis-ci.org/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). From 591fa34b4f8698c7901c24e41db5f0b0a57fc0b5 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 03:44:55 -0700 Subject: [PATCH 016/109] fix email address format in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 449bb62..fb83d87 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ version='2.1', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', - author_email='v at sergeev.io', + author_email='v@sergeev.io', url='https://github.com/vsergeev/u-msgpack-python', py_modules=['umsgpack'], long_description="""u-msgpack-python is a lightweight `MessagePack `_ serializer and deserializer module written in pure Python, compatible with both Python 2 and Python 3, as well as CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest `MessagePack specification `_. In particular, it supports the new binary, UTF-8 string, and application-defined ext types. See https://github.com/vsergeev/u-msgpack-python for more information.""", From ed8bfbc3a32830e3c21974da66a72984e33e9284 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 25 Sep 2016 03:37:45 -0700 Subject: [PATCH 017/109] update version and docs to v2.2 --- CHANGELOG.md | 7 +++++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cb671a..944e1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +* Version 2.2 - 09/25/2015 + * Add `use_ordered_dict` option to unpacking functions for unpacking MessagePack maps into the `collections.OrderedDict` type. + * Add support for `bytearray` type to `unpackb`/`loads` functions. + * Fix intermittent unit test failures due to non-deterministic packing of dict test vectors. + * Fix several docstring examples and typos. + * Add license and unit test to source distribution packaging. + * Version 2.1 - 05/09/2015 * Improve array and map unpacking performance under Python 2. * Add module `__version__` attribute. diff --git a/setup.py b/setup.py index fb83d87..e3fcbb3 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.1', + version='2.2', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index 05855f0..4129513 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.1 - vsergeev at gmail +# u-msgpack-python v2.2 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.1 - vsergeev at gmail +u-msgpack-python v2.2 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -44,10 +44,10 @@ License: MIT """ -__version__ = "2.1" +__version__ = "2.2" "Module version string" -version = (2,1) +version = (2,2) "Module version tuple" import struct From 32f84aa36b1aff8d13bf2d49fc3edde9a5350876 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Mon, 17 Oct 2016 00:58:20 -0700 Subject: [PATCH 018/109] fix version 2.2 release date in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 944e1e8..38c182c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -* Version 2.2 - 09/25/2015 +* Version 2.2 - 09/25/2016 * Add `use_ordered_dict` option to unpacking functions for unpacking MessagePack maps into the `collections.OrderedDict` type. * Add support for `bytearray` type to `unpackb`/`loads` functions. * Fix intermittent unit test failures due to non-deterministic packing of dict test vectors. From 921ae159f8f0d26d8e15c76bef2079f81a3bff0d Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 14 Oct 2016 17:43:14 -0700 Subject: [PATCH 019/109] add contributors to changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38c182c..08137a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ * Fix intermittent unit test failures due to non-deterministic packing of dict test vectors. * Fix several docstring examples and typos. * Add license and unit test to source distribution packaging. + * Contributors + * Fairiz 'Fi' Azizi - df59af6 + * INADA Naoki - 23dbf70 + * Jack O'Connor - 7aa0d19 * Version 2.1 - 05/09/2015 * Improve array and map unpacking performance under Python 2. @@ -36,4 +40,5 @@ * Version 1.0 - 09/29/2013 * Initial release. - + * Contributors + * Eugene Ma - 496aaa5 From 9ea90b172d993908090055cdc47d5a80d35089cb Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 14 Oct 2016 18:05:14 -0700 Subject: [PATCH 020/109] add option to allow unpacking invalid utf8 strings resolves #2. --- README.md | 24 ++++++++++++++++++++---- test_umsgpack.py | 9 +++++++++ umsgpack.py | 24 ++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d63f17b..21dfb49 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,20 @@ OrderedDict([('compact', True), ('schema', 0)]) >>> ``` +### Invalid UTF-8 Strings + +The unpacking functions provide an `allow_invalid_utf8` option to unpack MessagePack strings with invalid UTF-8 into the `umsgpack.InvalidString` type, instead of throwing an exception. The `umsgpack.InvalidString` type is a subclass of `bytes`, and can be used like any other `bytes` object. + +``` python +>>> # Attempt to unpack invalid UTF-8 string +... umsgpack.unpackb(b'\xa4\x80\x01\x02\x03') +... +umsgpack.InvalidStringException: unpacked string is invalid utf-8 +>>> umsgpack.unpackb(b'\xa4\x80\x01\x02\x03', allow_invalid_utf8=True) +b'\x80\x01\x02\x03' +>>> +``` + ### Compatibility Mode The compatibility mode supports the "raw" bytes MessagePack type from the [old specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md). When the module-wide `compatibility` option is enabled, both unicode strings and bytes will be serialized into the "raw" MessagePack type, and the "raw" MessagePack type will be deserialized into bytes. @@ -216,13 +230,15 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a ``` * `InvalidStringException`: Invalid UTF-8 string encountered during unpacking. - String bytes are strictly decoded with UTF-8. This exception is thrown if UTF-8 decoding of string bytes fails. + String bytes are strictly decoded with UTF-8. This exception is thrown if + UTF-8 decoding of string bytes fails. Use the `allow_invalid_utf8` option + to unpack invalid MessagePack strings into byte strings. ``` python - # Attempt to unpack the string b"\x80\x81" + # Attempt to unpack invalid UTF-8 string >>> umsgpack.unpackb(b"\xa2\x80\x81") ... - umsgpack.InvalidStringException: unpacked string is not utf-8 + umsgpack.InvalidStringException: unpacked string is invalid utf-8 >>> ``` @@ -268,7 +284,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * Python 3 * `str` type objects are packed into, and unpacked from, the msgpack `string` format * `bytes` type objects are packed into, and unpacked from, the msgpack `binary` format -* The msgpack string format is strictly decoded with UTF-8 -- an exception is thrown if the string bytes cannot be decoded into a valid UTF-8 string +* The msgpack string format is strictly decoded with UTF-8 — an exception is thrown if the string bytes cannot be decoded into a valid UTF-8 string, unless the `allow_invalid_utf8` option is enabled * The msgpack array format is unpacked into a Python list, unless it is the key of a map, in which case it is unpacked into a Python tuple * Python tuples and lists are both packed into the msgpack array format * Python float types are packed into the msgpack float32 or float64 format depending on the system's `sys.float_info` diff --git a/test_umsgpack.py b/test_umsgpack.py index 81ada20..f96f1fe 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -224,6 +224,7 @@ # These are the only global variables that should be exported by umsgpack exported_vars_test_vector = [ "Ext", + "InvalidString", "PackException", "UnpackException", "UnsupportedTypeException", @@ -332,6 +333,14 @@ def test_unpack_compatibility(self): umsgpack.compatibility = False + def test_unpack_invalid_string(self): + # Use last unpack exception test vector (an invalid string) + (_, data, _) = unpack_exception_test_vectors[-1] + + obj = umsgpack.unpackb(data, allow_invalid_utf8=True) + self.assertTrue(isinstance(obj, umsgpack.InvalidString)) + self.assertEqual(obj, b"\x80") + def test_unpack_ordered_dict(self): # Use last composite test vector (a map) (_, obj, data) = composite_test_vectors[-1] diff --git a/umsgpack.py b/umsgpack.py index 4129513..2481477 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -129,6 +129,10 @@ def __str__(self): s += ")" return s +class InvalidString(bytes): + """Subclass of bytes to hold invalid UTF-8 strings.""" + pass + ################################################################################ ### Exceptions ################################################################################ @@ -551,10 +555,13 @@ def _unpack_string(code, fp, options): if compatibility: return _read_except(fp, length) + data = _read_except(fp, length) try: - return bytes.decode(_read_except(fp, length), 'utf-8') + return bytes.decode(data, 'utf-8') except UnicodeDecodeError: - raise InvalidStringException("unpacked string is not utf-8") + if options.get("allow_invalid_utf8"): + return InvalidString(data) + raise InvalidStringException("unpacked string is invalid utf-8") def _unpack_binary(code, fp, options): if code == b'\xc4': @@ -655,6 +662,9 @@ def _unpack2(fp, **options): Kwargs: use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) Returns: A Python object. @@ -690,6 +700,9 @@ def _unpack3(fp, **options): Kwargs: use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) Returns: A Python object. @@ -726,6 +739,9 @@ def _unpackb2(s, **options): Kwargs: use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) Returns: A Python object. @@ -765,6 +781,10 @@ def _unpackb3(s, **options): Kwargs: use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + InvalidString, for access to the bytes + (default False) + Returns: A Python object. From 6f66430a52666cc465acc9501f9396078b8554a2 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 15 Oct 2016 00:41:34 -0700 Subject: [PATCH 021/109] add support for ext handlers to unpack --- umsgpack.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/umsgpack.py b/umsgpack.py index 2481477..987e02c 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -595,7 +595,14 @@ def _unpack_ext(code, fp, options): else: raise Exception("logic error, not ext: 0x%02x" % ord(code)) - return Ext(ord(_read_except(fp, 1)), _read_except(fp, length)) + ext = Ext(ord(_read_except(fp, 1)), _read_except(fp, length)) + + # Unpack with ext handler, if we have one + ext_handlers = options.get("ext_handlers") + if ext_handlers and ext.type in ext_handlers: + ext = ext_handlers[ext.type](ext) + + return ext def _unpack_array(code, fp, options): if (ord(code) & 0xf0) == 0x90: @@ -660,6 +667,9 @@ def _unpack2(fp, **options): fp: a .read()-supporting file-like object Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of @@ -698,6 +708,9 @@ def _unpack3(fp, **options): fp: a .read()-supporting file-like object Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of @@ -737,6 +750,9 @@ def _unpackb2(s, **options): s: a 'str' or 'bytearray' containing serialized MessagePack bytes Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of @@ -779,6 +795,9 @@ def _unpackb3(s, **options): s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of From eb2e053fe461d65849558da690ac2ab7077130a2 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 15 Oct 2016 01:20:45 -0700 Subject: [PATCH 022/109] add support for ext handlers to pack --- umsgpack.py | 125 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 83 insertions(+), 42 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index 987e02c..b5f17fa 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -216,7 +216,7 @@ class DuplicateKeyException(UnpackException): # chr(obj) has a str return type instead of bytes in Python 3, and # struct.pack(...) has the right return type in both versions. -def _pack_integer(obj, fp): +def _pack_integer(obj, fp, options): if obj < 0: if obj >= -32: fp.write(struct.pack("b", obj)) @@ -244,19 +244,19 @@ def _pack_integer(obj, fp): else: raise UnsupportedTypeException("huge unsigned int") -def _pack_nil(obj, fp): +def _pack_nil(obj, fp, options): fp.write(b"\xc0") -def _pack_boolean(obj, fp): +def _pack_boolean(obj, fp, options): fp.write(b"\xc3" if obj else b"\xc2") -def _pack_float(obj, fp): +def _pack_float(obj, fp, options): if _float_size == 64: fp.write(b"\xcb" + struct.pack(">d", obj)) else: fp.write(b"\xca" + struct.pack(">f", obj)) -def _pack_string(obj, fp): +def _pack_string(obj, fp, options): obj = obj.encode('utf-8') if len(obj) <= 31: fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) @@ -269,7 +269,7 @@ def _pack_string(obj, fp): else: raise UnsupportedTypeException("huge string") -def _pack_binary(obj, fp): +def _pack_binary(obj, fp, options): if len(obj) <= 2**8-1: fp.write(b"\xc4" + struct.pack("B", len(obj)) + obj) elif len(obj) <= 2**16-1: @@ -279,7 +279,7 @@ def _pack_binary(obj, fp): else: raise UnsupportedTypeException("huge binary string") -def _pack_oldspec_raw(obj, fp): +def _pack_oldspec_raw(obj, fp, options): if len(obj) <= 31: fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) elif len(obj) <= 2**16-1: @@ -289,7 +289,7 @@ def _pack_oldspec_raw(obj, fp): else: raise UnsupportedTypeException("huge raw string") -def _pack_ext(obj, fp): +def _pack_ext(obj, fp, options): if len(obj.data) == 1: fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) elif len(obj.data) == 2: @@ -309,7 +309,7 @@ def _pack_ext(obj, fp): else: raise UnsupportedTypeException("huge ext data") -def _pack_array(obj, fp): +def _pack_array(obj, fp, options): if len(obj) <= 15: fp.write(struct.pack("B", 0x90 | len(obj))) elif len(obj) <= 2**16-1: @@ -320,9 +320,9 @@ def _pack_array(obj, fp): raise UnsupportedTypeException("huge array") for e in obj: - pack(e, fp) + pack(e, fp, **options) -def _pack_map(obj, fp): +def _pack_map(obj, fp, options): if len(obj) <= 15: fp.write(struct.pack("B", 0x80 | len(obj))) elif len(obj) <= 2**16-1: @@ -333,13 +333,13 @@ def _pack_map(obj, fp): raise UnsupportedTypeException("huge array") for k,v in obj.items(): - pack(k, fp) - pack(v, fp) + pack(k, fp, **options) + pack(v, fp, **options) ######################################## # Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type -def _pack2(obj, fp): +def _pack2(obj, fp, **options): """ Serialize a Python object into MessagePack bytes. @@ -347,6 +347,11 @@ def _pack2(obj, fp): obj: a Python object fp: a .write()-supporting file-like object + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + Returns: None. @@ -359,36 +364,46 @@ def _pack2(obj, fp): >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) >>> """ - global compatibility + ext_handlers = options.get("ext_handlers") + if obj is None: - _pack_nil(obj, fp) + _pack_nil(obj, fp, options) + elif ext_handlers and obj.__class__ in ext_handlers: + _pack_ext(ext_handlers[obj.__class__](obj), fp, options) elif isinstance(obj, bool): - _pack_boolean(obj, fp) + _pack_boolean(obj, fp, options) elif isinstance(obj, int) or isinstance(obj, long): - _pack_integer(obj, fp) + _pack_integer(obj, fp, options) elif isinstance(obj, float): - _pack_float(obj, fp) + _pack_float(obj, fp, options) elif compatibility and isinstance(obj, unicode): - _pack_oldspec_raw(bytes(obj), fp) + _pack_oldspec_raw(bytes(obj), fp, options) elif compatibility and isinstance(obj, bytes): - _pack_oldspec_raw(obj, fp) + _pack_oldspec_raw(obj, fp, options) elif isinstance(obj, unicode): - _pack_string(obj, fp) + _pack_string(obj, fp, options) elif isinstance(obj, str): - _pack_binary(obj, fp) + _pack_binary(obj, fp, options) elif isinstance(obj, list) or isinstance(obj, tuple): - _pack_array(obj, fp) + _pack_array(obj, fp, options) elif isinstance(obj, dict): - _pack_map(obj, fp) + _pack_map(obj, fp, options) elif isinstance(obj, Ext): - _pack_ext(obj, fp) + _pack_ext(obj, fp, options) + elif ext_handlers: + # Linear search for superclass + t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) + if t: + _pack_ext(ext_handlers[t](obj), fp, options) + else: + raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) else: raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) # Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type -def _pack3(obj, fp): +def _pack3(obj, fp, **options): """ Serialize a Python object into MessagePack bytes. @@ -396,6 +411,11 @@ def _pack3(obj, fp): obj: a Python object fp: a .write()-supporting file-like object + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + Returns: None. @@ -410,38 +430,54 @@ def _pack3(obj, fp): """ global compatibility + ext_handlers = options.get("ext_handlers") + if obj is None: - _pack_nil(obj, fp) + _pack_nil(obj, fp, options) + elif ext_handlers and obj.__class__ in ext_handlers: + _pack_ext(ext_handlers[obj.__class__](obj), fp, options) elif isinstance(obj, bool): - _pack_boolean(obj, fp) + _pack_boolean(obj, fp, options) elif isinstance(obj, int): - _pack_integer(obj, fp) + _pack_integer(obj, fp, options) elif isinstance(obj, float): - _pack_float(obj, fp) + _pack_float(obj, fp, options) elif compatibility and isinstance(obj, str): - _pack_oldspec_raw(obj.encode('utf-8'), fp) + _pack_oldspec_raw(obj.encode('utf-8'), fp, options) elif compatibility and isinstance(obj, bytes): - _pack_oldspec_raw(obj, fp) + _pack_oldspec_raw(obj, fp, options) elif isinstance(obj, str): - _pack_string(obj, fp) + _pack_string(obj, fp, options) elif isinstance(obj, bytes): - _pack_binary(obj, fp) + _pack_binary(obj, fp, options) elif isinstance(obj, list) or isinstance(obj, tuple): - _pack_array(obj, fp) + _pack_array(obj, fp, options) elif isinstance(obj, dict): - _pack_map(obj, fp) + _pack_map(obj, fp, options) elif isinstance(obj, Ext): - _pack_ext(obj, fp) + _pack_ext(obj, fp, options) + elif ext_handlers: + # Linear search for superclass + t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) + if t: + _pack_ext(ext_handlers[t](obj), fp, options) + else: + raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) else: raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) -def _packb2(obj): +def _packb2(obj, **options): """ Serialize a Python object into MessagePack bytes. Args: obj: a Python object + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + Returns: A 'str' containing serialized MessagePack bytes. @@ -455,16 +491,21 @@ def _packb2(obj): >>> """ fp = io.BytesIO() - _pack2(obj, fp) + _pack2(obj, fp, **options) return fp.getvalue() -def _packb3(obj): +def _packb3(obj, **options): """ Serialize a Python object into MessagePack bytes. Args: obj: a Python object + Kwargs: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + Returns: A 'bytes' containing serialized MessagePack bytes. @@ -478,7 +519,7 @@ def _packb3(obj): >>> """ fp = io.BytesIO() - _pack3(obj, fp) + _pack3(obj, fp, **options) return fp.getvalue() ################################################################################ From 740ea271977966df9455b97c23a5f7ec73b9ec51 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 15 Oct 2016 01:59:40 -0700 Subject: [PATCH 023/109] add ext handlers tests to unit tests --- test_umsgpack.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index f96f1fe..3754e47 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -12,7 +12,7 @@ import struct import unittest import io -from collections import OrderedDict +from collections import OrderedDict, namedtuple import umsgpack @@ -221,6 +221,20 @@ [ "32-bit raw", b"b"*65536, b"\xdb\x00\x01\x00\x00" + b"b"*65536 ], ] +CustomType = namedtuple('CustomType', ['x', 'y', 'z']) + +ext_handlers = { + complex: lambda obj: umsgpack.Ext(0x20, struct.pack("ff", obj.real, obj.imag)), + CustomType: lambda obj: umsgpack.Ext(0x30, umsgpack.packb(list(obj))), + 0x20: lambda ext: complex(*struct.unpack("ff", ext.data)), + 0x30: lambda ext: CustomType(*umsgpack.unpackb(ext.data)), +} + +ext_handlers_test_vectors = [ + [ "complex", complex(1, 2), b"\xd7\x20\x00\x00\x80\x3f\x00\x00\x00\x40" ], + [ "custom type", CustomType(b"abc", 123, True), b"\xd7\x30\x93\xc4\x03\x61\x62\x63\x7b\xc3" ], +] + # These are the only global variables that should be exported by umsgpack exported_vars_test_vector = [ "Ext", @@ -368,6 +382,18 @@ def test_ext_exceptions(self): with self.assertRaises(TypeError): _ = umsgpack.Ext(0, u"unicode string") + def test_pack_ext_handler(self): + for (name, obj, data) in ext_handlers_test_vectors: + obj_repr = repr(obj) + print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.packb(obj, ext_handlers=ext_handlers), data) + + def test_unpack_ext_handler(self): + for (name, obj, data) in ext_handlers_test_vectors: + obj_repr = repr(obj) + print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.unpackb(data, ext_handlers=ext_handlers), obj) + def test_streaming_writer(self): # Try first composite test vector (_, obj, data) = composite_test_vectors[0] From 106a2175718fad5a67993ddad80e8d556902f091 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Mon, 17 Oct 2016 00:35:17 -0700 Subject: [PATCH 024/109] add hex prefix to data bytes in string representation of Ext --- umsgpack.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index b5f17fa..77d2eba 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -117,13 +117,7 @@ def __str__(self): String representation of this Ext object. """ s = "Ext Object (Type: 0x%02x, Data: " % self.type - for i in range(min(len(self.data), 8)): - if i > 0: - s += " " - if isinstance(self.data[i], int): - s += "%02x" % (self.data[i]) - else: - s += "%02x" % ord(self.data[i]) + s += " ".join(["0x%02x" % ord(self.data[i:i+1]) for i in xrange(min(len(self.data), 8))]) if len(self.data) > 8: s += " ..." s += ")" From 4a383321be47706fbbc09bffa2da9ba2ac955be8 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Mon, 17 Oct 2016 00:34:05 -0700 Subject: [PATCH 025/109] add ext handlers usage to readme and msgpack.org.md resolvs #19. --- README.md | 93 +++++++++++++++++++++++++++++++++++++++++++++++--- msgpack.org.md | 29 ++++++++++++++-- 2 files changed, 116 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 21dfb49..afd10cd 100644 --- a/README.md +++ b/README.md @@ -62,15 +62,16 @@ Streaming serialization with file-like objects: >>> ``` -Encoding and decoding an application-defined ext type: +Encoding and decoding an application-defined Ext type: ``` python # Create an Ext object with type 0x05 and data b"\x01\x02\x03" >>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) b'\x82\xadspecial stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> >>> bar = umsgpack.unpackb(_) >>> print(bar["special stuff"]) -Ext Object (Type: 0x05, Data: 01 02 03) +Ext Object (Type: 0x05, Data: 0x01 0x02 0x03) >>> bar["special stuff"].type 5 >>> bar["special stuff"].data @@ -78,8 +79,24 @@ b'\x01\x02\x03' >>> ``` -Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: +Encoding and decoding application-defined types with Ext handlers: +``` python +>>> umsgpack.packb([complex(1,2), datetime.datetime.now()], +... ext_handlers = { +... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... datetime.datetime: lambda obj: umsgpack.Ext(0x40, obj.strftime("%Y%m%dT%H:%M:%S.%f").encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xc7\x18@20161017T00:12:53.719377' +>>> umsgpack.unpackb(_, +... ext_handlers = { +... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: datetime.datetime.strptime(ext.data.decode(), "%Y%m%dT%H:%M:%S.%f"), +... }) +[(1+2j), datetime.datetime(2016, 10, 17, 0, 12, 53, 719377)] +>>> +``` +Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: ``` python >>> import umsgpack >>> umsgpack.dumps({u"compact": True, u"schema": 0}) @@ -97,6 +114,75 @@ Python standard library style names `dump`, `dumps`, `load`, `loads` are also av >>> ``` +## Ext Handlers + +The packing functions accept an optional `ext_handlers` dictionary that maps +custom types to callables that pack the type into an Ext object. The callable +should accept the custom type object as an argument and return a packed +`umsgpack.Ext` object. + +Example for packing `set`, `complex`, and `datetime.datetime` types into Ext +objects with type codes 0x20, 0x30, and 0x40, respectively: + +``` python +>>> umsgpack.packb([1, True, {"foo", 2}, complex(3, 4), datetime.datetime.now()], +... ext_handlers = { +... set: lambda obj: umsgpack.Ext(0x20, umsgpack.packb(list(obj))), +... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... datetime.datetime: lambda obj: umsgpack.Ext(0x40, obj.strftime("%Y%m%dT%H:%M:%S.%f").encode()), +... }) +b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xc7\x18@20161015T02:28:35.666425' +>>> +``` + +Similarly, the unpacking functions accept an optional `ext_handlers` dictionary +that maps Ext type codes to callables that unpack the Ext into a custom object. +The callable should accept a `umsgpack.Ext` object as an argument and return an +unpacked custom type object. + +Example for unpacking Ext objects with type codes 0x20, 0x30, and 0x40, into +`set`, `complex`, and `datetime.datetime` typed objects, respectively: + +``` python +>>> umsgpack.unpackb(b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@' \ +... b'\xc7\x18@20161015T02:28:35.666425', +... ext_handlers = { +... 0x20: lambda ext: set(umsgpack.unpackb(ext.data)), +... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: datetime.datetime.strptime(ext.data.decode(), "%Y%m%dT%H:%M:%S.%f"), +... }) +[1, True, {'foo', 2}, (3+4j), datetime.datetime(2016, 10, 15, 2, 28, 35, 666425)] +>>> +``` + +Example for packing and unpacking a custom class: + +``` python +class Point(object): + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z + + def __str__(self): + return "Point({}, {}, {})".format(self.x, self.y, self.z) + + def pack(self): + return struct.pack(">iii", self.x, self.y, self.z) + + @staticmethod + def unpack(data): + return Point(*struct.unpack(">iii", data)) + +# Pack +obj = Point(1,2,3) +data = umsgpack.packb(obj, ext_handlers = {Point: lambda obj: umsgpack.Ext(0x10, obj.pack())}) + +# Unpack +obj = umsgpack.unpackb(data, ext_handlers = {0x10: lambda ext: Point.unpack(ext.data)}) +print(obj) # -> Point(1, 2, 3) +``` + ## Streaming Serialization and Deserialization The streaming `pack()`/`dump()` and `unpack()`/`load()` functions allow packing and unpacking objects directly to and from a stream, respectively. Streaming may be necessary when unpacking serialized bytes whose size is unknown in advance, or it may be more convenient and efficient when working directly with stream objects (e.g. files or stream sockets). @@ -303,4 +389,3 @@ $ pypy3 test_umsgpack.py ## License u-msgpack-python is MIT licensed. See the included `LICENSE` file for more details. - diff --git a/msgpack.org.md b/msgpack.org.md index 5cf18b0..02ff9ac 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -69,9 +69,10 @@ Encoding and decoding an application-defined ext type: ... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) b'\x82\xadspecial stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> >>> bar = umsgpack.unpackb(_) >>> print(bar["special stuff"]) -Ext Object (Type: 0x05, Data: 01 02 03) +Ext Object (Type: 0x05, Data: 0x01 0x02 0x03) >>> bar["special stuff"].type 5 >>> bar["special stuff"].data @@ -79,6 +80,31 @@ b'\x01\x02\x03' >>> ``` +Encoding and decoding application-defined types with Ext handlers: +``` python +>>> umsgpack.packb([complex(1,2), datetime.datetime.now()], +... ext_handlers = { +... complex: lambda obj: umsgpack.Ext(0x30, +... struct.pack("ff", obj.real, obj.imag)), +... datetime.datetime: lambda obj: umsgpack.Ext(0x40, +... obj.strftime("%Y%m%dT%H:%M:%S.%f").encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xc7\x18@20161017T00:12:53.7' +b'19377' +>>> umsgpack.unpackb(_, +... ext_handlers = { +... 0x30: lambda ext: +... complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: +... datetime.datetime.strptime( +... ext.data.decode(), +... "%Y%m%dT%H:%M:%S.%f" +... ), +... }) +[(1+2j), datetime.datetime(2016, 10, 17, 0, 12, 53, 719377)] +>>> +``` + Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: @@ -106,4 +132,3 @@ See the [project page](https://github.com/vsergeev/u-msgpack-python) for more in ## License u-msgpack-python is MIT licensed. See the included `LICENSE` file for more details. - From fbd5b76aa176dd664937f60713c43e8605bf4ab1 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Mon, 17 Oct 2016 00:34:38 -0700 Subject: [PATCH 026/109] improve formatting in readme and msgpack.org.md --- README.md | 35 +++++++++++++++++------------------ msgpack.org.md | 30 ++++++++++++++---------------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index afd10cd..65a8f36 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, and application-defined ext types. -u-msgpack-python is currently distributed on PyPI: https://pypi.python.org/pypi/u-msgpack-python and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) +u-msgpack-python is currently distributed on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py). ## Installation @@ -27,7 +27,7 @@ Basic Example: ``` python >>> import umsgpack >>> umsgpack.packb({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.unpackb(_) {u'compact': True, u'schema': 0} >>> @@ -35,13 +35,13 @@ Basic Example: A more complicated example: ``` python ->>> umsgpack.packb( [1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", - u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345] ) -'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01' -'\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb' -'@\x00\xfc\xd3Z\x85\x87\x94' +>>> umsgpack.packb([1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", \ +... u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345]) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' >>> umsgpack.unpackb(_) -[1, True, False, 4294967295, {u'foo': '\x80\x01\x02', +[1, True, False, 4294967295, {u'foo': b'\x80\x01\x02', \ u'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] >>> ``` @@ -62,19 +62,19 @@ Streaming serialization with file-like objects: >>> ``` -Encoding and decoding an application-defined Ext type: +Encoding and decoding a raw Ext type: ``` python -# Create an Ext object with type 0x05 and data b"\x01\x02\x03" ->>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") ->>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) -b'\x82\xadspecial stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> # Create an Ext object with type 0x05 and data b"\x01\x02\x03" +... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") +>>> umsgpack.packb({u"stuff": foo, u"awesome": True}) +b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' >>> >>> bar = umsgpack.unpackb(_) ->>> print(bar["special stuff"]) +>>> print(bar['stuff']) Ext Object (Type: 0x05, Data: 0x01 0x02 0x03) ->>> bar["special stuff"].type +>>> bar['stuff'].type 5 ->>> bar["special stuff"].data +>>> bar['stuff'].data b'\x01\x02\x03' >>> ``` @@ -98,9 +98,8 @@ b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xc7\x18@20161017T00:12:53.719377' Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: ``` python ->>> import umsgpack >>> umsgpack.dumps({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.loads(_) {u'compact': True, u'schema': 0} >>> diff --git a/msgpack.org.md b/msgpack.org.md index 02ff9ac..c03a35b 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -27,7 +27,7 @@ Basic Example: ``` python >>> import umsgpack >>> umsgpack.packb({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.unpackb(_) {u'compact': True, u'schema': 0} >>> @@ -36,13 +36,13 @@ Basic Example: A more complicated example: ``` python >>> umsgpack.packb( - [1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", - u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345] ) -'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01' -'\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb' -'@\x00\xfc\xd3Z\x85\x87\x94' +... [1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", +... u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345] ) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' >>> umsgpack.unpackb(_) -[1, True, False, 4294967295, {u'foo': '\x80\x01\x02', +[1, True, False, 4294967295, {u'foo': b'\x80\x01\x02', \ u'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] >>> ``` @@ -63,19 +63,19 @@ Streaming serialization with file-like objects: >>> ``` -Encoding and decoding an application-defined ext type: +Encoding and decoding a raw Ext type: ``` python >>> # Create an Ext object with type 0x05 and data b"\x01\x02\x03" ... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") ->>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) -b'\x82\xadspecial stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> umsgpack.packb({u"stuff": foo, u"awesome": True}) +b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' >>> >>> bar = umsgpack.unpackb(_) ->>> print(bar["special stuff"]) +>>> print(bar['stuff']) Ext Object (Type: 0x05, Data: 0x01 0x02 0x03) ->>> bar["special stuff"].type +>>> bar['stuff'].type 5 ->>> bar["special stuff"].data +>>> bar['stuff'].data b'\x01\x02\x03' >>> ``` @@ -107,11 +107,9 @@ b'19377' Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: - ``` python ->>> import umsgpack >>> umsgpack.dumps({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.loads(_) {u'compact': True, u'schema': 0} >>> From 73ab43dacbca5f7b441aacd69bdc6c6296fb3b40 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 19 Oct 2016 05:50:37 -0700 Subject: [PATCH 027/109] change encoded wording to serialized in InsufficientDataException docstring --- README.md | 4 ++-- umsgpack.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 65a8f36..f446c12 100644 --- a/README.md +++ b/README.md @@ -297,10 +297,10 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a >>> ``` -* `InsufficientDataException`: Insufficient data to unpack the encoded object. +* `InsufficientDataException`: Insufficient data to unpack the serialized object. ``` python - # Attempt to unpack a cut-off encoded 32-bit unsigned int + # Attempt to unpack a cut-off serialized 32-bit unsigned int >>> umsgpack.unpackb(b"\xce\xff\xff\xff") ... umsgpack.InsufficientDataException diff --git a/umsgpack.py b/umsgpack.py index 77d2eba..1d6fd2d 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -146,7 +146,7 @@ class UnsupportedTypeException(PackException): # Unpacking error class InsufficientDataException(UnpackException): - "Insufficient data to unpack the encoded object." + "Insufficient data to unpack the serialized object." pass class InvalidStringException(UnpackException): "Invalid UTF-8 string encountered during unpacking." @@ -716,7 +716,7 @@ def _unpack2(fp, **options): Raises: InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. + Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. ReservedCodeException(UnpackException): @@ -757,7 +757,7 @@ def _unpack3(fp, **options): Raises: InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. + Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. ReservedCodeException(UnpackException): @@ -801,7 +801,7 @@ def _unpackb2(s, **options): TypeError: Packed data type is neither 'str' nor 'bytearray'. InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. + Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. ReservedCodeException(UnpackException): @@ -846,7 +846,7 @@ def _unpackb3(s, **options): TypeError: Packed data type is neither 'bytes' nor 'bytearray'. InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. + Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. ReservedCodeException(UnpackException): From 3b980fe162782d6e2f43d266ce6077df5c2fd13b Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 19 Oct 2016 05:51:48 -0700 Subject: [PATCH 028/109] improve wording in readme and msgpack.org.md --- README.md | 8 ++++---- msgpack.org.md | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f446c12..f05149c 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Streaming serialization with file-like objects: >>> ``` -Encoding and decoding a raw Ext type: +Serializing and deserializing a raw Ext type: ``` python >>> # Create an Ext object with type 0x05 and data b"\x01\x02\x03" ... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") @@ -79,7 +79,7 @@ b'\x01\x02\x03' >>> ``` -Encoding and decoding application-defined types with Ext handlers: +Serializing and deserializing application-defined types with Ext handlers: ``` python >>> umsgpack.packb([complex(1,2), datetime.datetime.now()], ... ext_handlers = { @@ -339,7 +339,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * `UnhashableKeyException`: Unhashable key encountered during map unpacking. The packed map cannot be unpacked into a Python dictionary. - Python dictionaries only support keys that are instances of `collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a MessagePack encoding, it cannot be unpacked into a valid Python dictionary. + Python dictionaries only support keys that are instances of `collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a MessagePack serialization, it cannot be unpacked into a valid Python dictionary. ``` python # Attempt to unpack { {} : False } @@ -351,7 +351,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * `DuplicateKeyException`: Duplicate key encountered during map unpacking. - Python dictionaries do not support duplicate keys, but MessagePack maps may be encoded with duplicate keys. + Python dictionaries do not support duplicate keys, but MessagePack maps may be serialized with duplicate keys. ``` python # Attempt to unpack { 1: True, 1: False } diff --git a/msgpack.org.md b/msgpack.org.md index c03a35b..c94d371 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -63,7 +63,7 @@ Streaming serialization with file-like objects: >>> ``` -Encoding and decoding a raw Ext type: +Serializing and deserializing a raw Ext type: ``` python >>> # Create an Ext object with type 0x05 and data b"\x01\x02\x03" ... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") @@ -80,7 +80,7 @@ b'\x01\x02\x03' >>> ``` -Encoding and decoding application-defined types with Ext handlers: +Serializing and deserializing application-defined types with Ext handlers: ``` python >>> umsgpack.packb([complex(1,2), datetime.datetime.now()], ... ext_handlers = { From a2f8804746570d1929586f267aee7a850bd0f747 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Mon, 17 Oct 2016 01:12:00 -0700 Subject: [PATCH 029/109] add username to license copyright --- LICENSE | 2 +- umsgpack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 5b6471d..7e330d5 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2013-2016 Ivan (Vanya) A. Sergeev + Copyright (c) 2013-2016 vsergeev / Ivan (Vanya) A. Sergeev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/umsgpack.py b/umsgpack.py index 1d6fd2d..04787c6 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -10,7 +10,7 @@ # # MIT License # -# Copyright (c) 2013-2016 Ivan (Vanya) A. Sergeev +# Copyright (c) 2013-2016 vsergeev / Ivan (Vanya) A. Sergeev # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From b7bdf036e77e7a5e0b7660e2002f7474ba794722 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Mon, 17 Oct 2016 01:13:27 -0700 Subject: [PATCH 030/109] update version and changelog to v2.3.0 --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08137a1..87ca470 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +* Version 2.3.0 - 10/19/2016 + * Add `ext_handlers` option to packing and unpacking functions to support application-defined Ext packing and unpacking hooks. + * Add `allow_invalid_utf8` option to unpacking functions to allow unpacking of invalid UTF-8 strings. + * Add hexadecimal prefix to data bytes in string representation of Ext objects. + * Change version number to semantic versioning. + * Version 2.2 - 09/25/2016 * Add `use_ordered_dict` option to unpacking functions for unpacking MessagePack maps into the `collections.OrderedDict` type. * Add support for `bytearray` type to `unpackb`/`loads` functions. diff --git a/setup.py b/setup.py index e3fcbb3..b326345 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.2', + version='2.3.0', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index 04787c6..c0647e0 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.2 - v at sergeev.io +# u-msgpack-python v2.3.0 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.2 - v at sergeev.io +u-msgpack-python v2.3.0 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -44,10 +44,10 @@ License: MIT """ -__version__ = "2.2" +__version__ = "2.3.0" "Module version string" -version = (2,2) +version = (2,3,0) "Module version tuple" import struct From 4c461edb61f8658c7d0ad1c58c1551490b319651 Mon Sep 17 00:00:00 2001 From: Fabien Fleutot Date: Wed, 25 Jan 2017 16:56:07 +0100 Subject: [PATCH 031/109] implement hash special method in Ext class allowing Ext objects to be used as map keys. --- umsgpack.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/umsgpack.py b/umsgpack.py index c0647e0..5c40359 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -123,6 +123,13 @@ def __str__(self): s += ")" return s + def __hash__(self): + """ + Provide a hash of this Ext object. + """ + return hash((self.type, self.data)) + + class InvalidString(bytes): """Subclass of bytes to hold invalid UTF-8 strings.""" pass From 5f53bcf28dc4c70c7e259e2fa47ec4e479a8d031 Mon Sep 17 00:00:00 2001 From: "Yuhang(Steven) Wang" Date: Fri, 25 Nov 2016 14:20:11 -0600 Subject: [PATCH 032/109] make codebase pep8 compliant --- test_umsgpack.py | 408 +++++++++++++++++++++++++++++------------------ umsgpack.py | 202 ++++++++++++++--------- 2 files changed, 383 insertions(+), 227 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index 3754e47..c16f3a1 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -18,207 +18,275 @@ single_test_vectors = [ # None - [ "nil", None, b"\xc0" ], + ["nil", None, b"\xc0"], # Booleans - [ "bool false", False, b"\xc2" ], - [ "bool true", True, b"\xc3" ], + ["bool false", False, b"\xc2"], + ["bool true", True, b"\xc3"], # + 7-bit uint - [ "7-bit uint", 0x00, b"\x00" ], - [ "7-bit uint", 0x10, b"\x10" ], - [ "7-bit uint", 0x7f, b"\x7f" ], + ["7-bit uint", 0x00, b"\x00"], + ["7-bit uint", 0x10, b"\x10"], + ["7-bit uint", 0x7f, b"\x7f"], # - 5-bit int - [ "5-bit sint", -1, b"\xff" ], - [ "5-bit sint", -16, b"\xf0" ], - [ "5-bit sint", -32, b"\xe0" ], + ["5-bit sint", -1, b"\xff"], + ["5-bit sint", -16, b"\xf0"], + ["5-bit sint", -32, b"\xe0"], # 8-bit uint - [ "8-bit uint", 0x80, b"\xcc\x80" ], - [ "8-bit uint", 0xf0, b"\xcc\xf0" ], - [ "8-bit uint", 0xff, b"\xcc\xff" ], + ["8-bit uint", 0x80, b"\xcc\x80"], + ["8-bit uint", 0xf0, b"\xcc\xf0"], + ["8-bit uint", 0xff, b"\xcc\xff"], # 16-bit uint - [ "16-bit uint", 0x100, b"\xcd\x01\x00" ], - [ "16-bit uint", 0x2000, b"\xcd\x20\x00" ], - [ "16-bit uint", 0xffff, b"\xcd\xff\xff" ], + ["16-bit uint", 0x100, b"\xcd\x01\x00"], + ["16-bit uint", 0x2000, b"\xcd\x20\x00"], + ["16-bit uint", 0xffff, b"\xcd\xff\xff"], # 32-bit uint - [ "32-bit uint", 0x10000, b"\xce\x00\x01\x00\x00" ], - [ "32-bit uint", 0x200000, b"\xce\x00\x20\x00\x00" ], - [ "32-bit uint", 0xffffffff, b"\xce\xff\xff\xff\xff" ], + ["32-bit uint", 0x10000, b"\xce\x00\x01\x00\x00"], + ["32-bit uint", 0x200000, b"\xce\x00\x20\x00\x00"], + ["32-bit uint", 0xffffffff, b"\xce\xff\xff\xff\xff"], # 64-bit uint - [ "64-bit uint", 0x100000000, b"\xcf\x00\x00\x00\x01\x00\x00\x00\x00" ], - [ "64-bit uint", 0x200000000000, b"\xcf\x00\x00\x20\x00\x00\x00\x00\x00" ], - [ "64-bit uint", 0xffffffffffffffff, b"\xcf\xff\xff\xff\xff\xff\xff\xff\xff" ], + ["64-bit uint", 0x100000000, b"\xcf\x00\x00\x00\x01\x00\x00\x00\x00"], + ["64-bit uint", 0x200000000000, b"\xcf\x00\x00\x20\x00\x00\x00\x00\x00"], + ["64-bit uint", 0xffffffffffffffff, b"\xcf\xff\xff\xff\xff\xff\xff\xff\xff"], # 8-bit int - [ "8-bit int", -33, b"\xd0\xdf" ], - [ "8-bit int", -100, b"\xd0\x9c" ], - [ "8-bit int", -128, b"\xd0\x80" ], + ["8-bit int", -33, b"\xd0\xdf"], + ["8-bit int", -100, b"\xd0\x9c"], + ["8-bit int", -128, b"\xd0\x80"], # 16-bit int - [ "16-bit int", -129, b"\xd1\xff\x7f" ], - [ "16-bit int", -2000, b"\xd1\xf8\x30" ], - [ "16-bit int", -32768, b"\xd1\x80\x00" ], + ["16-bit int", -129, b"\xd1\xff\x7f"], + ["16-bit int", -2000, b"\xd1\xf8\x30"], + ["16-bit int", -32768, b"\xd1\x80\x00"], # 32-bit int - [ "32-bit int", -32769, b"\xd2\xff\xff\x7f\xff" ], - [ "32-bit int", -1000000000, b"\xd2\xc4\x65\x36\x00" ], - [ "32-bit int", -2147483648, b"\xd2\x80\x00\x00\x00" ], + ["32-bit int", -32769, b"\xd2\xff\xff\x7f\xff"], + ["32-bit int", -1000000000, b"\xd2\xc4\x65\x36\x00"], + ["32-bit int", -2147483648, b"\xd2\x80\x00\x00\x00"], # 64-bit int - [ "64-bit int", -2147483649, b"\xd3\xff\xff\xff\xff\x7f\xff\xff\xff" ], - [ "64-bit int", -1000000000000000002, b"\xd3\xf2\x1f\x49\x4c\x58\x9b\xff\xfe" ], - [ "64-bit int", -9223372036854775808, b"\xd3\x80\x00\x00\x00\x00\x00\x00\x00" ], + ["64-bit int", -2147483649, b"\xd3\xff\xff\xff\xff\x7f\xff\xff\xff"], + ["64-bit int", -1000000000000000002, b"\xd3\xf2\x1f\x49\x4c\x58\x9b\xff\xfe"], + ["64-bit int", -9223372036854775808, b"\xd3\x80\x00\x00\x00\x00\x00\x00\x00"], # 64-bit float - [ "64-bit float", 0.0, b"\xcb\x00\x00\x00\x00\x00\x00\x00\x00" ], - [ "64-bit float", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00" ], - [ "64-bit float", float(10**35), b"\xcb\x47\x33\x42\x61\x72\xc7\x4d\x82" ], + ["64-bit float", 0.0, b"\xcb\x00\x00\x00\x00\x00\x00\x00\x00"], + ["64-bit float", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00"], + ["64-bit float", float(10**35), b"\xcb\x47\x33\x42\x61\x72\xc7\x4d\x82"], # Fixstr String - [ "fix string", u"", b"\xa0" ], - [ "fix string", u"a", b"\xa1\x61" ], - [ "fix string", u"abc", b"\xa3\x61\x62\x63" ], - [ "fix string", u"a"*31, b"\xbf" + b"\x61"*31 ], + ["fix string", u"", b"\xa0"], + ["fix string", u"a", b"\xa1\x61"], + ["fix string", u"abc", b"\xa3\x61\x62\x63"], + ["fix string", u"a" * 31, b"\xbf" + b"\x61" * 31], # 8-bit String - [ "8-bit string", u"b"*32, b"\xd9\x20" + b"b"*32 ], - [ "8-bit string", u"c"*100, b"\xd9\x64" + b"c"*100 ], - [ "8-bit string", u"d"*255, b"\xd9\xff" + b"d"*255 ], + ["8-bit string", u"b" * 32, b"\xd9\x20" + b"b" * 32], + ["8-bit string", u"c" * 100, b"\xd9\x64" + b"c" * 100], + ["8-bit string", u"d" * 255, b"\xd9\xff" + b"d" * 255], # 16-bit String - [ "16-bit string", u"b"*256, b"\xda\x01\x00" + b"b"*256 ], - [ "16-bit string", u"c"*65535, b"\xda\xff\xff" + b"c"*65535 ], + ["16-bit string", u"b" * 256, b"\xda\x01\x00" + b"b" * 256], + ["16-bit string", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535], # 32-bit String - [ "32-bit string", u"b"*65536, b"\xdb\x00\x01\x00\x00" + b"b"*65536 ], + ["32-bit string", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536], # Wide character String - [ "wide char string", u"Allagbé", b"\xa8Allagb\xc3\xa9" ], - [ "wide char string", u"По оживлённым берегам", b"\xd9\x28\xd0\x9f\xd0\xbe\x20\xd0\xbe\xd0\xb6\xd0\xb8\xd0\xb2\xd0\xbb\xd1\x91\xd0\xbd\xd0\xbd\xd1\x8b\xd0\xbc\x20\xd0\xb1\xd0\xb5\xd1\x80\xd0\xb5\xd0\xb3\xd0\xb0\xd0\xbc" ], + ["wide char string", u"Allagbé", b"\xa8Allagb\xc3\xa9"], + ["wide char string", u"По оживлённым берегам", + b"\xd9\x28\xd0\x9f\xd0\xbe\x20\xd0\xbe\xd0\xb6\xd0\xb8\xd0\xb2\xd0\xbb\xd1\x91\xd0\xbd\xd0\xbd\xd1\x8b\xd0\xbc\x20\xd0\xb1\xd0\xb5\xd1\x80\xd0\xb5\xd0\xb3\xd0\xb0\xd0\xbc"], # 8-bit Binary - [ "8-bit binary", b"\x80"*1, b"\xc4\x01" + b"\x80"*1 ], - [ "8-bit binary", b"\x80"*32, b"\xc4\x20" + b"\x80"*32 ], - [ "8-bit binary", b"\x80"*255, b"\xc4\xff" + b"\x80"*255 ], + ["8-bit binary", b"\x80" * 1, b"\xc4\x01" + b"\x80" * 1], + ["8-bit binary", b"\x80" * 32, b"\xc4\x20" + b"\x80" * 32], + ["8-bit binary", b"\x80" * 255, b"\xc4\xff" + b"\x80" * 255], # 16-bit Binary - [ "16-bit binary", b"\x80"*256, b"\xc5\x01\x00" + b"\x80"*256 ], + ["16-bit binary", b"\x80" * 256, b"\xc5\x01\x00" + b"\x80" * 256], # 32-bit Binary - [ "32-bit binary", b"\x80"*65536, b"\xc6\x00\x01\x00\x00" + b"\x80"*65536 ], + ["32-bit binary", b"\x80" * 65536, b"\xc6\x00\x01\x00\x00" + b"\x80" * 65536], # Fixext 1 - [ "fixext 1", umsgpack.Ext(0x05, b"\x80"*1), b"\xd4\x05" + b"\x80"*1 ], + ["fixext 1", umsgpack.Ext(0x05, b"\x80" * 1), b"\xd4\x05" + b"\x80" * 1], # Fixext 2 - [ "fixext 2", umsgpack.Ext(0x05, b"\x80"*2), b"\xd5\x05" + b"\x80"*2 ], + ["fixext 2", umsgpack.Ext(0x05, b"\x80" * 2), b"\xd5\x05" + b"\x80" * 2], # Fixext 4 - [ "fixext 4", umsgpack.Ext(0x05, b"\x80"*4), b"\xd6\x05" + b"\x80"*4 ], + ["fixext 4", umsgpack.Ext(0x05, b"\x80" * 4), b"\xd6\x05" + b"\x80" * 4], # Fixext 8 - [ "fixext 8", umsgpack.Ext(0x05, b"\x80"*8), b"\xd7\x05" + b"\x80"*8 ], + ["fixext 8", umsgpack.Ext(0x05, b"\x80" * 8), b"\xd7\x05" + b"\x80" * 8], # Fixext 16 - [ "fixext 16", umsgpack.Ext(0x05, b"\x80"*16), b"\xd8\x05" + b"\x80"*16 ], + ["fixext 16", umsgpack.Ext(0x05, b"\x80" * 16), + b"\xd8\x05" + b"\x80" * 16], # 8-bit Ext - [ "8-bit ext", umsgpack.Ext(0x05, b"\x80"*255), b"\xc7\xff\x05" + b"\x80"*255 ], + ["8-bit ext", umsgpack.Ext(0x05, b"\x80" * 255), + b"\xc7\xff\x05" + b"\x80" * 255], # 16-bit Ext - [ "16-bit ext", umsgpack.Ext(0x05, b"\x80"*256), b"\xc8\x01\x00\x05" + b"\x80"*256 ], + ["16-bit ext", umsgpack.Ext(0x05, b"\x80" * 256), + b"\xc8\x01\x00\x05" + b"\x80" * 256], # 32-bit Ext - [ "32-bit ext", umsgpack.Ext(0x05, b"\x80"*65536), b"\xc9\x00\x01\x00\x00\x05" + b"\x80"*65536 ], + ["32-bit ext", umsgpack.Ext(0x05, b"\x80" * 65536), + b"\xc9\x00\x01\x00\x00\x05" + b"\x80" * 65536], # Empty Array - [ "empty array", [], b"\x90" ], + ["empty array", [], b"\x90"], # Empty Map - [ "empty map", {}, b"\x80" ], + ["empty map", {}, b"\x80"], ] composite_test_vectors = [ # Fix Array - [ "fix array", [ 5, u"abc", True ], b"\x93\x05\xa3\x61\x62\x63\xc3" ], + ["fix array", [5, u"abc", True], + b"\x93\x05\xa3\x61\x62\x63\xc3"], # 16-bit Array - [ "16-bit array", [ 0x05 ]*16, b"\xdc\x00\x10" + b"\x05"*16 ], - [ "16-bit array", [ 0x05 ]*65535, b"\xdc\xff\xff" + b"\x05"*65535 ], + ["16-bit array", [0x05] * 16, + b"\xdc\x00\x10" + b"\x05" * 16], + ["16-bit array", [0x05] * 65535, + b"\xdc\xff\xff" + b"\x05" * 65535], # 32-bit Array - [ "32-bit array", [ 0x05 ]*65536, b"\xdd\x00\x01\x00\x00" + b"\x05"*65536 ], + ["32-bit array", [0x05] * 65536, + b"\xdd\x00\x01\x00\x00" + b"\x05" * 65536], # Fix Map - [ "fix map", OrderedDict([(1, True), (2, u"abc"), (3, b"\x80")]), b"\x83\x01\xc3\x02\xa3\x61\x62\x63\x03\xc4\x01\x80" ], - [ "fix map", { u"abc" : 5 }, b"\x81\xa3\x61\x62\x63\x05" ], - [ "fix map", { b"\x80" : 0xffff }, b"\x81\xc4\x01\x80\xcd\xff\xff" ], - [ "fix map", { True : None }, b"\x81\xc3\xc0" ], + ["fix map", OrderedDict([(1, True), (2, u"abc"), (3, b"\x80")]), + b"\x83\x01\xc3\x02\xa3\x61\x62\x63\x03\xc4\x01\x80"], + ["fix map", {u"abc": 5}, + b"\x81\xa3\x61\x62\x63\x05"], + ["fix map", {b"\x80": 0xffff}, + b"\x81\xc4\x01\x80\xcd\xff\xff"], + ["fix map", {True: None}, + b"\x81\xc3\xc0"], # 16-bit Map - [ "16-bit map", OrderedDict([(k, 0x05) for k in range(16)]), b"\xde\x00\x10" + b"".join([struct.pack("B", i) + b"\x05" for i in range(16)])], - [ "16-bit map", OrderedDict([(k, 0x05) for k in range(6000)]), b"\xde\x17\x70" + b"".join([struct.pack("B", i) + b"\x05" for i in range(128)]) + b"".join([b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + b"".join([b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)]) ], + ["16-bit map", OrderedDict([(k, 0x05) for k in range(16)]), + b"\xde\x00\x10" + b"".join([struct.pack("B", i) + b"\x05" for i in range(16)])], + ["16-bit map", OrderedDict([(k, 0x05) for k in range(6000)]), + b"\xde\x17\x70" + b"".join([struct.pack("B", i) + b"\x05" for i in range(128)]) + + b"".join([b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + + b"".join([b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)])], # Complex Array - [ "complex array", [ True, 0x01, umsgpack.Ext(0x03, b"foo"), 0xff, OrderedDict([(1, False), (2, u"abc")]), b"\x80", [1, 2, 3], u"abc" ], b"\x98\xc3\x01\xc7\x03\x03\x66\x6f\x6f\xcc\xff\x82\x01\xc2\x02\xa3\x61\x62\x63\xc4\x01\x80\x93\x01\x02\x03\xa3\x61\x62\x63" ], + ["complex array", [True, 0x01, umsgpack.Ext(0x03, b"foo"), 0xff, + OrderedDict([(1, False), (2, u"abc")]), b"\x80", + [1, 2, 3], u"abc"], + b"\x98\xc3\x01\xc7\x03\x03\x66\x6f\x6f\xcc\xff\x82\x01\xc2\x02\xa3\x61\x62\x63\xc4\x01\x80\x93\x01\x02\x03\xa3\x61\x62\x63"], # Complex Map - [ "complex map", OrderedDict([(1, [OrderedDict([(1, 2), (3, 4)]), {}]), (2, 1), (3, [False, u"def"]), (4, OrderedDict([(0x100000000, u"a"), (0xffffffff, u"b")]))]), b"\x84\x01\x92\x82\x01\x02\x03\x04\x80\x02\x01\x03\x92\xc2\xa3\x64\x65\x66\x04\x82\xcf\x00\x00\x00\x01\x00\x00\x00\x00\xa1\x61\xce\xff\xff\xff\xff\xa1\x62" ], + ["complex map", OrderedDict([(1, [OrderedDict([(1, 2), (3, 4)]), {}]), + (2, 1), (3, [False, u"def"]), + (4, OrderedDict([(0x100000000, u"a"), + (0xffffffff, u"b")]))]), + b"\x84\x01\x92\x82\x01\x02\x03\x04\x80\x02\x01\x03\x92\xc2\xa3\x64\x65\x66\x04\x82\xcf\x00\x00\x00\x01\x00\x00\x00\x00\xa1\x61\xce\xff\xff\xff\xff\xa1\x62"], # Map with Tuple Keys - [ "map with tuple keys", OrderedDict([((u"foo", False, 3), True), ((3e6, -5), u"def")]), b"\x82\x93\xa3\x66\x6f\x6f\xc2\x03\xc3\x92\xcb\x41\x46\xe3\x60\x00\x00\x00\x00\xfb\xa3\x64\x65\x66" ], + ["map with tuple keys", OrderedDict([((u"foo", False, 3), True), + ((3e6, -5), u"def")]), + b"\x82\x93\xa3\x66\x6f\x6f\xc2\x03\xc3\x92\xcb\x41\x46\xe3\x60\x00\x00\x00\x00\xfb\xa3\x64\x65\x66"], # Map with Complex Tuple Keys - [ "map with complex tuple keys", {(u"foo", (1,2,3), 3) : -5}, b"\x81\x93\xa3\x66\x6f\x6f\x93\x01\x02\x03\x03\xfb" ] + ["map with complex tuple keys", {(u"foo", (1, 2, 3), 3): -5}, + b"\x81\x93\xa3\x66\x6f\x6f\x93\x01\x02\x03\x03\xfb"] ] pack_exception_test_vectors = [ # Unsupported type exception - [ "unsupported type", set([1,2,3]), umsgpack.UnsupportedTypeException ], - [ "unsupported type", -2**(64-1)-1, umsgpack.UnsupportedTypeException ], - [ "unsupported type", 2**64, umsgpack.UnsupportedTypeException ], + ["unsupported type", set([1, 2, 3]), umsgpack.UnsupportedTypeException], + ["unsupported type", -2**(64 - 1) - 1, umsgpack.UnsupportedTypeException], + ["unsupported type", 2**64, umsgpack.UnsupportedTypeException], ] unpack_exception_test_vectors = [ # Type errors - [ "type error unpack unicode string", u"\x01", TypeError ], - [ "type error unpack boolean", True, TypeError ], + ["type error unpack unicode string", u"\x01", TypeError], + ["type error unpack boolean", True, TypeError], # Insufficient data to unpack object - [ "insufficient data 8-bit uint", b"\xcc", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit uint", b"\xcd\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit uint", b"\xce\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 64-bit uint", b"\xcf\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit int", b"\xd0", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit int", b"\xd1\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit int", b"\xd2\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 64-bit int", b"\xd3\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit float", b"\xca\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 64-bit float", b"\xcb\xff", umsgpack.InsufficientDataException ], - [ "insufficient data fixstr", b"\xa1", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit string", b"\xd9", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit string", b"\xd9\x01", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit string", b"\xda\x01\x00", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit string", b"\xdb\x00\x01\x00\x00", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit binary", b"\xc4", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit binary", b"\xc4\x01", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit binary", b"\xc5\x01\x00", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit binary", b"\xc6\x00\x01\x00\x00", umsgpack.InsufficientDataException ], - [ "insufficient data fixarray", b"\x91", umsgpack.InsufficientDataException ], - [ "insufficient data fixarray", b"\x92\xc2", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit array", b"\xdc\x00\xf0\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit array", b"\xdd\x00\x01\x00\x00\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data fixmap", b"\x81", umsgpack.InsufficientDataException ], - [ "insufficient data fixmap", b"\x82\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit map", b"\xde\x00\xf0\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit map", b"\xdf\x00\x01\x00\x00\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 1", b"\xd4", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 1", b"\xd4\x05", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 2", b"\xd5\x05\x01", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 4", b"\xd6\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 8", b"\xd7\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 16", b"\xd8\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data ext 8-bit", b"\xc7\x05\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data ext 16-bit", b"\xc8\x01\x00\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data ext 32-bit", b"\xc9\x00\x01\x00\x00\x05\x01\x02\x03", umsgpack.InsufficientDataException ], + ["insufficient data 8-bit uint", b"\xcc", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit uint", b"\xcd\xff", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit uint", b"\xce\xff", + umsgpack.InsufficientDataException], + ["insufficient data 64-bit uint", b"\xcf\xff", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit int", b"\xd0", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit int", b"\xd1\xff", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit int", b"\xd2\xff", + umsgpack.InsufficientDataException], + ["insufficient data 64-bit int", b"\xd3\xff", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit float", b"\xca\xff", + umsgpack.InsufficientDataException], + ["insufficient data 64-bit float", b"\xcb\xff", + umsgpack.InsufficientDataException], + ["insufficient data fixstr", b"\xa1", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit string", b"\xd9", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit string", b"\xd9\x01", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit string", b"\xda\x01\x00", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit string", b"\xdb\x00\x01\x00\x00", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit binary", b"\xc4", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit binary", b"\xc4\x01", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit binary", b"\xc5\x01\x00", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit binary", b"\xc6\x00\x01\x00\x00", + umsgpack.InsufficientDataException], + ["insufficient data fixarray", b"\x91", + umsgpack.InsufficientDataException], + ["insufficient data fixarray", b"\x92\xc2", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit array", b"\xdc\x00\xf0\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit array", b"\xdd\x00\x01\x00\x00\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data fixmap", b"\x81", + umsgpack.InsufficientDataException], + ["insufficient data fixmap", b"\x82\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit map", b"\xde\x00\xf0\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit map", b"\xdf\x00\x01\x00\x00\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data fixext 1", b"\xd4", + umsgpack.InsufficientDataException], + ["insufficient data fixext 1", b"\xd4\x05", + umsgpack.InsufficientDataException], + ["insufficient data fixext 2", b"\xd5\x05\x01", + umsgpack.InsufficientDataException], + ["insufficient data fixext 4", b"\xd6\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data fixext 8", b"\xd7\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data fixext 16", b"\xd8\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data ext 8-bit", b"\xc7\x05\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data ext 16-bit", b"\xc8\x01\x00\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data ext 32-bit", b"\xc9\x00\x01\x00\x00\x05\x01\x02\x03", + umsgpack.InsufficientDataException], # Unhashable key { 1 : True, { 1 : 1 } : False } - [ "unhashable key", b"\x82\x01\xc3\x81\x01\x01\xc2", umsgpack.UnhashableKeyException ], + ["unhashable key", b"\x82\x01\xc3\x81\x01\x01\xc2", + umsgpack.UnhashableKeyException], # Unhashable key { [ 1, 2, {} ] : True } - [ "unhashable key", b"\x81\x93\x01\x02\x80\xc3", umsgpack.UnhashableKeyException ], + ["unhashable key", b"\x81\x93\x01\x02\x80\xc3", + umsgpack.UnhashableKeyException], # Key duplicate { 1 : True, 1 : False } - [ "duplicate key", b"\x82\x01\xc3\x01\xc2", umsgpack.DuplicateKeyException ], + ["duplicate key", b"\x82\x01\xc3\x01\xc2", + umsgpack.DuplicateKeyException], # Reserved code (0xc1) - [ "reserved code", b"\xc1", umsgpack.ReservedCodeException ], + ["reserved code", b"\xc1", + umsgpack.ReservedCodeException], # Invalid string (non utf-8) - [ "invalid string", b"\xa1\x80", umsgpack.InvalidStringException ], + ["invalid string", b"\xa1\x80", + umsgpack.InvalidStringException], ] compatibility_test_vectors = [ # Fix Raw - [ "fix raw", b"", b"\xa0" ], - [ "fix raw", u"", b"\xa0" ], - [ "fix raw", b"a", b"\xa1\x61" ], - [ "fix raw", u"a", b"\xa1\x61" ], - [ "fix raw", b"abc", b"\xa3\x61\x62\x63" ], - [ "fix raw", u"abc", b"\xa3\x61\x62\x63" ], - [ "fix raw", b"a"*31, b"\xbf" + b"\x61"*31 ], - [ "fix raw", u"a"*31, b"\xbf" + b"\x61"*31 ], + ["fix raw", b"", b"\xa0"], + ["fix raw", u"", b"\xa0"], + ["fix raw", b"a", b"\xa1\x61"], + ["fix raw", u"a", b"\xa1\x61"], + ["fix raw", b"abc", b"\xa3\x61\x62\x63"], + ["fix raw", u"abc", b"\xa3\x61\x62\x63"], + ["fix raw", b"a" * 31, b"\xbf" + b"\x61" * 31], + ["fix raw", u"a" * 31, b"\xbf" + b"\x61" * 31], # 16-bit Raw - [ "16-bit raw", u"b"*32, b"\xda\x00\x20" + b"b"*32 ], - [ "16-bit raw", b"b"*32, b"\xda\x00\x20" + b"b"*32 ], - [ "16-bit raw", u"b"*256, b"\xda\x01\x00" + b"b"*256 ], - [ "16-bit raw", b"b"*256, b"\xda\x01\x00" + b"b"*256 ], - [ "16-bit raw", u"c"*65535, b"\xda\xff\xff" + b"c"*65535 ], - [ "16-bit raw", b"c"*65535, b"\xda\xff\xff" + b"c"*65535 ], + ["16-bit raw", u"b" * 32, b"\xda\x00\x20" + b"b" * 32], + ["16-bit raw", b"b" * 32, b"\xda\x00\x20" + b"b" * 32], + ["16-bit raw", u"b" * 256, b"\xda\x01\x00" + b"b" * 256], + ["16-bit raw", b"b" * 256, b"\xda\x01\x00" + b"b" * 256], + ["16-bit raw", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535], + ["16-bit raw", b"c" * 65535, b"\xda\xff\xff" + b"c" * 65535], # 32-bit Raw - [ "32-bit raw", u"b"*65536, b"\xdb\x00\x01\x00\x00" + b"b"*65536 ], - [ "32-bit raw", b"b"*65536, b"\xdb\x00\x01\x00\x00" + b"b"*65536 ], + ["32-bit raw", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536], + ["32-bit raw", b"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536], ] CustomType = namedtuple('CustomType', ['x', 'y', 'z']) @@ -231,8 +299,9 @@ } ext_handlers_test_vectors = [ - [ "complex", complex(1, 2), b"\xd7\x20\x00\x00\x80\x3f\x00\x00\x00\x40" ], - [ "custom type", CustomType(b"abc", 123, True), b"\xd7\x30\x93\xc4\x03\x61\x62\x63\x7b\xc3" ], + ["complex", complex(1, 2), b"\xd7\x20\x00\x00\x80\x3f\x00\x00\x00\x40"], + ["custom type", CustomType(b"abc", 123, True), + b"\xd7\x30\x93\xc4\x03\x61\x62\x63\x7b\xc3"], ] # These are the only global variables that should be exported by umsgpack @@ -261,32 +330,42 @@ "compatibility", ] -################################################################################ +########################################################################## + class TestUmsgpack(unittest.TestCase): + def test_pack_single(self): for (name, obj, data) in single_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.packb(obj), data) def test_pack_composite(self): for (name, obj, data) in composite_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.packb(obj), data) def test_pack_exceptions(self): for (name, obj, exception) in pack_exception_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + with self.assertRaises(exception): umsgpack.packb(obj) def test_unpack_single(self): for (name, obj, data) in single_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + unpacked = umsgpack.unpackb(data) # In Python2, we have both int and long integer types, but which @@ -294,7 +373,8 @@ def test_unpack_single(self): if sys.version_info[0] == 2: # Allow both {int,long} -> unpackb -> {int,long} if isinstance(obj, int) or isinstance(obj, long): - self.assertTrue(isinstance(unpacked, int) or isinstance(unpacked, long)) + self.assertTrue(isinstance(unpacked, int) or + isinstance(unpacked, long)) else: self.assertTrue(isinstance(unpacked, type(obj))) # In Python3, we only have the int integer type @@ -306,12 +386,15 @@ def test_unpack_single(self): def test_unpack_composite(self): for (name, obj, data) in composite_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.unpackb(data), obj) def test_unpack_exceptions(self): for (name, data, exception) in unpack_exception_test_vectors: print("\tTesting %s" % name) + with self.assertRaises(exception): umsgpack.unpackb(data) @@ -320,7 +403,9 @@ def test_pack_compatibility(self): for (name, obj, data) in compatibility_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.packb(obj), data) umsgpack.compatibility = False @@ -330,7 +415,9 @@ def test_unpack_compatibility(self): for (name, obj, data) in compatibility_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + unpacked = umsgpack.unpackb(data) # Encoded raw should always unpack to bytes in compatibility mode, @@ -385,14 +472,20 @@ def test_ext_exceptions(self): def test_pack_ext_handler(self): for (name, obj, data) in ext_handlers_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) - self.assertEqual(umsgpack.packb(obj, ext_handlers=ext_handlers), data) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj, ext_handlers=ext_handlers) + self.assertEqual(packed, data) def test_unpack_ext_handler(self): for (name, obj, data) in ext_handlers_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) - self.assertEqual(umsgpack.unpackb(data, ext_handlers=ext_handlers), obj) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + unpacked = umsgpack.unpackb(data, ext_handlers=ext_handlers) + self.assertEqual(unpacked, obj) def test_streaming_writer(self): # Try first composite test vector @@ -409,13 +502,16 @@ def test_streaming_reader(self): def test_namespacing(self): # Get a list of global variables from umsgpack module - exported_vars = list(filter(lambda x: not x.startswith("_"), dir(umsgpack))) + exported_vars = list(filter(lambda x: not x.startswith("_"), + dir(umsgpack))) # Ignore imports - exported_vars = list(filter(lambda x: x != "struct" and x != "collections" and x != "sys" and x != "io" and x != "xrange", exported_vars)) + exported_vars = list(filter(lambda x: x != "struct" and x != "collections" and x != + "sys" and x != "io" and x != "xrange", exported_vars)) self.assertTrue(len(exported_vars) == len(exported_vars_test_vector)) for var in exported_vars_test_vector: self.assertTrue(var in exported_vars) + if __name__ == '__main__': unittest.main() diff --git a/umsgpack.py b/umsgpack.py index 5c40359..7169b6f 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -43,21 +43,21 @@ License: MIT """ +import struct +import collections +import sys +import io __version__ = "2.3.0" "Module version string" -version = (2,3,0) +version = (2, 3, 0) "Module version tuple" -import struct -import collections -import sys -import io -################################################################################ -### Ext Class -################################################################################ +############################################################################## +# Ext Class +############################################################################## # Extension type for application-defined types and data class Ext: @@ -117,7 +117,8 @@ def __str__(self): String representation of this Ext object. """ s = "Ext Object (Type: 0x%02x, Data: " % self.type - s += " ".join(["0x%02x" % ord(self.data[i:i+1]) for i in xrange(min(len(self.data), 8))]) + s += " ".join(["0x%02x" % ord(self.data[i:i + 1]) + for i in xrange(min(len(self.data), 8))]) if len(self.data) > 8: s += " ..." s += ")" @@ -134,50 +135,64 @@ class InvalidString(bytes): """Subclass of bytes to hold invalid UTF-8 strings.""" pass -################################################################################ -### Exceptions -################################################################################ +############################################################################## +# Exceptions +############################################################################## + # Base Exception classes class PackException(Exception): "Base class for exceptions encountered during packing." pass + + class UnpackException(Exception): "Base class for exceptions encountered during unpacking." pass + # Packing error class UnsupportedTypeException(PackException): "Object type not supported for packing." pass + # Unpacking error class InsufficientDataException(UnpackException): "Insufficient data to unpack the serialized object." pass + + class InvalidStringException(UnpackException): "Invalid UTF-8 string encountered during unpacking." pass + + class ReservedCodeException(UnpackException): "Reserved code encountered during unpacking." pass + + class UnhashableKeyException(UnpackException): """ Unhashable key encountered during map unpacking. The serialized map cannot be deserialized into a Python dictionary. """ pass + + class DuplicateKeyException(UnpackException): "Duplicate key encountered during map unpacking." pass + # Backwards compatibility KeyNotPrimitiveException = UnhashableKeyException KeyDuplicateException = DuplicateKeyException -################################################################################ -### Exported Functions and Globals -################################################################################ +############################################################################# +# Exported Functions and Glob +############################################################################# # Exported functions and variables, set up in __init() pack = None @@ -208,88 +223,96 @@ class DuplicateKeyException(UnpackException): >>> """ -################################################################################ -### Packing -################################################################################ +############################################################################## +# Packing +############################################################################## # You may notice struct.pack("B", obj) instead of the simpler chr(obj) in the # code below. This is to allow for seamless Python 2 and 3 compatibility, as # chr(obj) has a str return type instead of bytes in Python 3, and # struct.pack(...) has the right return type in both versions. + def _pack_integer(obj, fp, options): if obj < 0: if obj >= -32: fp.write(struct.pack("b", obj)) - elif obj >= -2**(8-1): + elif obj >= -2**(8 - 1): fp.write(b"\xd0" + struct.pack("b", obj)) - elif obj >= -2**(16-1): + elif obj >= -2**(16 - 1): fp.write(b"\xd1" + struct.pack(">h", obj)) - elif obj >= -2**(32-1): + elif obj >= -2**(32 - 1): fp.write(b"\xd2" + struct.pack(">i", obj)) - elif obj >= -2**(64-1): + elif obj >= -2**(64 - 1): fp.write(b"\xd3" + struct.pack(">q", obj)) else: raise UnsupportedTypeException("huge signed int") else: if obj <= 127: fp.write(struct.pack("B", obj)) - elif obj <= 2**8-1: + elif obj <= 2**8 - 1: fp.write(b"\xcc" + struct.pack("B", obj)) - elif obj <= 2**16-1: + elif obj <= 2**16 - 1: fp.write(b"\xcd" + struct.pack(">H", obj)) - elif obj <= 2**32-1: + elif obj <= 2**32 - 1: fp.write(b"\xce" + struct.pack(">I", obj)) - elif obj <= 2**64-1: + elif obj <= 2**64 - 1: fp.write(b"\xcf" + struct.pack(">Q", obj)) else: raise UnsupportedTypeException("huge unsigned int") + def _pack_nil(obj, fp, options): fp.write(b"\xc0") + def _pack_boolean(obj, fp, options): fp.write(b"\xc3" if obj else b"\xc2") + def _pack_float(obj, fp, options): if _float_size == 64: fp.write(b"\xcb" + struct.pack(">d", obj)) else: fp.write(b"\xca" + struct.pack(">f", obj)) + def _pack_string(obj, fp, options): obj = obj.encode('utf-8') if len(obj) <= 31: fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**8-1: + elif len(obj) <= 2**8 - 1: fp.write(b"\xd9" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16-1: + elif len(obj) <= 2**16 - 1: fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32-1: + elif len(obj) <= 2**32 - 1: fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) else: raise UnsupportedTypeException("huge string") + def _pack_binary(obj, fp, options): - if len(obj) <= 2**8-1: + if len(obj) <= 2**8 - 1: fp.write(b"\xc4" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16-1: + elif len(obj) <= 2**16 - 1: fp.write(b"\xc5" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32-1: + elif len(obj) <= 2**32 - 1: fp.write(b"\xc6" + struct.pack(">I", len(obj)) + obj) else: raise UnsupportedTypeException("huge binary string") + def _pack_oldspec_raw(obj, fp, options): if len(obj) <= 31: fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**16-1: + elif len(obj) <= 2**16 - 1: fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32-1: + elif len(obj) <= 2**32 - 1: fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) else: raise UnsupportedTypeException("huge raw string") + def _pack_ext(obj, fp, options): if len(obj.data) == 1: fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) @@ -301,21 +324,25 @@ def _pack_ext(obj, fp, options): fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data) elif len(obj.data) == 16: fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**8-1: - fp.write(b"\xc7" + struct.pack("BB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**16-1: - fp.write(b"\xc8" + struct.pack(">HB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**32-1: - fp.write(b"\xc9" + struct.pack(">IB", len(obj.data), obj.type & 0xff) + obj.data) + elif len(obj.data) <= 2**8 - 1: + fp.write(b"\xc7" + + struct.pack("BB", len(obj.data), obj.type & 0xff) + obj.data) + elif len(obj.data) <= 2**16 - 1: + fp.write(b"\xc8" + + struct.pack(">HB", len(obj.data), obj.type & 0xff) + obj.data) + elif len(obj.data) <= 2**32 - 1: + fp.write(b"\xc9" + + struct.pack(">IB", len(obj.data), obj.type & 0xff) + obj.data) else: raise UnsupportedTypeException("huge ext data") + def _pack_array(obj, fp, options): if len(obj) <= 15: fp.write(struct.pack("B", 0x90 | len(obj))) - elif len(obj) <= 2**16-1: + elif len(obj) <= 2**16 - 1: fp.write(b"\xdc" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32-1: + elif len(obj) <= 2**32 - 1: fp.write(b"\xdd" + struct.pack(">I", len(obj))) else: raise UnsupportedTypeException("huge array") @@ -323,22 +350,24 @@ def _pack_array(obj, fp, options): for e in obj: pack(e, fp, **options) + def _pack_map(obj, fp, options): if len(obj) <= 15: fp.write(struct.pack("B", 0x80 | len(obj))) - elif len(obj) <= 2**16-1: + elif len(obj) <= 2**16 - 1: fp.write(b"\xde" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32-1: + elif len(obj) <= 2**32 - 1: fp.write(b"\xdf" + struct.pack(">I", len(obj))) else: raise UnsupportedTypeException("huge array") - for k,v in obj.items(): + for k, v in obj.items(): pack(k, fp, **options) pack(v, fp, **options) ######################################## + # Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type def _pack2(obj, fp, **options): """ @@ -399,10 +428,12 @@ def _pack2(obj, fp, **options): if t: _pack_ext(ext_handlers[t](obj), fp, options) else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + raise UnsupportedTypeException( + "unsupported type: %s" % str(type(obj))) else: raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + # Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type def _pack3(obj, fp, **options): """ @@ -463,9 +494,12 @@ def _pack3(obj, fp, **options): if t: _pack_ext(ext_handlers[t](obj), fp, options) else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + raise UnsupportedTypeException( + "unsupported type: %s" % str(type(obj))) else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + raise UnsupportedTypeException( + "unsupported type: %s" % str(type(obj))) + def _packb2(obj, **options): """ @@ -495,6 +529,7 @@ def _packb2(obj, **options): _pack2(obj, fp, **options) return fp.getvalue() + def _packb3(obj, **options): """ Serialize a Python object into MessagePack bytes. @@ -523,9 +558,10 @@ def _packb3(obj, **options): _pack3(obj, fp, **options) return fp.getvalue() -################################################################################ -### Unpacking -################################################################################ +############################################################################# +# Unpacking +############################################################################# + def _read_except(fp, n): data = fp.read(n) @@ -533,6 +569,7 @@ def _read_except(fp, n): raise InsufficientDataException() return data + def _unpack_integer(code, fp, options): if (ord(code) & 0xe0) == 0xe0: return struct.unpack("b", code)[0] @@ -556,16 +593,21 @@ def _unpack_integer(code, fp, options): return struct.unpack(">Q", _read_except(fp, 8))[0] raise Exception("logic error, not int: 0x%02x" % ord(code)) + def _unpack_reserved(code, fp, options): if code == b'\xc1': - raise ReservedCodeException("encountered reserved code: 0x%02x" % ord(code)) - raise Exception("logic error, not reserved code: 0x%02x" % ord(code)) + raise ReservedCodeException( + "encountered reserved code: 0x%02x" % ord(code)) + raise Exception( + "logic error, not reserved code: 0x%02x" % ord(code)) + def _unpack_nil(code, fp, options): if code == b'\xc0': return None raise Exception("logic error, not nil: 0x%02x" % ord(code)) + def _unpack_boolean(code, fp, options): if code == b'\xc2': return False @@ -573,6 +615,7 @@ def _unpack_boolean(code, fp, options): return True raise Exception("logic error, not boolean: 0x%02x" % ord(code)) + def _unpack_float(code, fp, options): if code == b'\xca': return struct.unpack(">f", _read_except(fp, 4))[0] @@ -580,6 +623,7 @@ def _unpack_float(code, fp, options): return struct.unpack(">d", _read_except(fp, 8))[0] raise Exception("logic error, not float: 0x%02x" % ord(code)) + def _unpack_string(code, fp, options): if (ord(code) & 0xe0) == 0xa0: length = ord(code) & ~0xe0 @@ -605,6 +649,7 @@ def _unpack_string(code, fp, options): return InvalidString(data) raise InvalidStringException("unpacked string is invalid utf-8") + def _unpack_binary(code, fp, options): if code == b'\xc4': length = struct.unpack("B", _read_except(fp, 1))[0] @@ -617,6 +662,7 @@ def _unpack_binary(code, fp, options): return _read_except(fp, length) + def _unpack_ext(code, fp, options): if code == b'\xd4': length = 1 @@ -646,6 +692,7 @@ def _unpack_ext(code, fp, options): return ext + def _unpack_array(code, fp, options): if (ord(code) & 0xf0) == 0x90: length = (ord(code) & ~0xf0) @@ -658,11 +705,13 @@ def _unpack_array(code, fp, options): return [_unpack(fp, options) for i in xrange(length)] + def _deep_list_to_tuple(obj): if isinstance(obj, list): return tuple([_deep_list_to_tuple(e) for e in obj]) return obj + def _unpack_map(code, fp, options): if (ord(code) & 0xf0) == 0x80: length = (ord(code) & ~0xf0) @@ -673,7 +722,8 @@ def _unpack_map(code, fp, options): else: raise Exception("logic error, not map: 0x%02x" % ord(code)) - d = {} if not options.get('use_ordered_dict') else collections.OrderedDict() + d = {} if not options.get('use_ordered_dict') \ + else collections.OrderedDict() for _ in xrange(length): # Unpack key k = _unpack(fp, options) @@ -682,9 +732,11 @@ def _unpack_map(code, fp, options): # Attempt to convert list into a hashable tuple k = _deep_list_to_tuple(k) elif not isinstance(k, collections.Hashable): - raise UnhashableKeyException("encountered unhashable key: %s, %s" % (str(k), str(type(k)))) + raise UnhashableKeyException( + "encountered unhashable key: %s, %s" % (str(k), str(type(k)))) elif k in d: - raise DuplicateKeyException("encountered duplicate key: %s, %s" % (str(k), str(type(k)))) + raise DuplicateKeyException( + "encountered duplicate key: %s, %s" % (str(k), str(type(k)))) # Unpack value v = _unpack(fp, options) @@ -692,15 +744,18 @@ def _unpack_map(code, fp, options): try: d[k] = v except TypeError: - raise UnhashableKeyException("encountered unhashable key: %s" % str(k)) + raise UnhashableKeyException( + "encountered unhashable key: %s" % str(k)) return d + def _unpack(fp, options): code = _read_except(fp, 1) return _unpack_dispatch_table[code](code, fp, options) ######################################## + def _unpack2(fp, **options): """ Deserialize MessagePack bytes into a Python object. @@ -742,6 +797,7 @@ def _unpack2(fp, **options): """ return _unpack(fp, options) + def _unpack3(fp, **options): """ Deserialize MessagePack bytes into a Python object. @@ -783,6 +839,7 @@ def _unpack3(fp, **options): """ return _unpack(fp, options) + # For Python 2, expects a str object def _unpackb2(s, **options): """ @@ -828,6 +885,7 @@ def _unpackb2(s, **options): raise TypeError("packed data must be type 'str' or 'bytearray'") return _unpack(io.BytesIO(s), options) + # For Python 3, expects a bytes object def _unpackb3(s, **options): """ @@ -873,9 +931,10 @@ def _unpackb3(s, **options): raise TypeError("packed data must be type 'bytes' or 'bytearray'") return _unpack(io.BytesIO(s), options) -################################################################################ -### Module Initialization -################################################################################ +############################################################################# +# Module Initialization +############################################################################# + def __init(): global pack @@ -925,16 +984,16 @@ def __init(): _unpack_dispatch_table = {} # Fix uint - for code in range(0, 0x7f+1): + for code in range(0, 0x7f + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer # Fix map - for code in range(0x80, 0x8f+1): + for code in range(0x80, 0x8f + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_map # Fix array - for code in range(0x90, 0x9f+1): + for code in range(0x90, 0x9f + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_array # Fix str - for code in range(0xa0, 0xbf+1): + for code in range(0xa0, 0xbf + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string # Nil _unpack_dispatch_table[b'\xc0'] = _unpack_nil @@ -944,25 +1003,25 @@ def __init(): _unpack_dispatch_table[b'\xc2'] = _unpack_boolean _unpack_dispatch_table[b'\xc3'] = _unpack_boolean # Bin - for code in range(0xc4, 0xc6+1): + for code in range(0xc4, 0xc6 + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_binary # Ext - for code in range(0xc7, 0xc9+1): + for code in range(0xc7, 0xc9 + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext # Float _unpack_dispatch_table[b'\xca'] = _unpack_float _unpack_dispatch_table[b'\xcb'] = _unpack_float # Uint - for code in range(0xcc, 0xcf+1): + for code in range(0xcc, 0xcf + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer # Int - for code in range(0xd0, 0xd3+1): + for code in range(0xd0, 0xd3 + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer # Fixext - for code in range(0xd4, 0xd8+1): + for code in range(0xd4, 0xd8 + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext # String - for code in range(0xd9, 0xdb+1): + for code in range(0xd9, 0xdb + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string # Array _unpack_dispatch_table[b'\xdc'] = _unpack_array @@ -971,7 +1030,8 @@ def __init(): _unpack_dispatch_table[b'\xde'] = _unpack_map _unpack_dispatch_table[b'\xdf'] = _unpack_map # Negative fixint - for code in range(0xe0, 0xff+1): + for code in range(0xe0, 0xff + 1): _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + __init() From bdeee2064b38847ed40e969a078fd2162f9e9329 Mon Sep 17 00:00:00 2001 From: Fabien Fleutot Date: Fri, 10 Feb 2017 11:21:47 +0100 Subject: [PATCH 033/109] add packing option to force float precision resolves #27. --- README.md | 16 +++++++++++++++- test_umsgpack.py | 14 ++++++++++++++ umsgpack.py | 30 +++++++++++++++++++++++++----- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f05149c..e12c86a 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,21 @@ b'\x80\x01\x02\x03' >>> ``` -### Compatibility Mode +### Float Precision + +The packing functions provide a `force_float_precision` option to force packing of floats into the specified precision: `"single"` for IEEE-754 single-precision floats, or `"double"` for IEEE-754 double-precision floats. + +``` python +>>> # Force float packing to single-precision floats +... umsgpack.packb(2.5, force_float_precision="single") +b'\xca@ \x00\x00' +>>> # Force float packing to double-precision floats +... umsgpack.packb(2.5, force_float_precision="double") +b'\xcb@\x04\x00\x00\x00\x00\x00\x00' +>>> +``` + +### Old Specification Compatibility Mode The compatibility mode supports the "raw" bytes MessagePack type from the [old specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md). When the module-wide `compatibility` option is enabled, both unicode strings and bytes will be serialized into the "raw" MessagePack type, and the "raw" MessagePack type will be deserialized into bytes. diff --git a/test_umsgpack.py b/test_umsgpack.py index c16f3a1..a25b5b8 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -289,6 +289,11 @@ ["32-bit raw", b"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536], ] +float_precision_test_vectors = [ + ["float precision single", 2.5, b"\xca\x40\x20\x00\x00"], + ["float precision double", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00"], +] + CustomType = namedtuple('CustomType', ['x', 'y', 'z']) ext_handlers = { @@ -487,6 +492,15 @@ def test_unpack_ext_handler(self): unpacked = umsgpack.unpackb(data, ext_handlers=ext_handlers) self.assertEqual(unpacked, obj) + def test_pack_force_float_precision(self): + for ((name, obj, data), precision) in zip(float_precision_test_vectors, ["single", "double"]): + obj_repr = repr(obj) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj, force_float_precision=precision) + self.assertEqual(packed, data) + def test_streaming_writer(self): # Try first composite test vector (_, obj, data) = composite_test_vectors[0] diff --git a/umsgpack.py b/umsgpack.py index 7169b6f..54da44c 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -271,10 +271,14 @@ def _pack_boolean(obj, fp, options): def _pack_float(obj, fp, options): - if _float_size == 64: + float_precision = options.get('force_float_precision', _float_precision) + + if float_precision == "double": fp.write(b"\xcb" + struct.pack(">d", obj)) - else: + elif float_precision == "single": fp.write(b"\xca" + struct.pack(">f", obj)) + else: + raise ValueError("invalid float precision") def _pack_string(obj, fp, options): @@ -381,6 +385,10 @@ def _pack2(obj, fp, **options): ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. Returns: None. @@ -447,6 +455,10 @@ def _pack3(obj, fp, **options): ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. Returns: None. @@ -512,6 +524,10 @@ def _packb2(obj, **options): ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. Returns: A 'str' containing serialized MessagePack bytes. @@ -541,6 +557,10 @@ def _packb3(obj, **options): ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats. Returns: A 'bytes' containing serialized MessagePack bytes. @@ -946,7 +966,7 @@ def __init(): global load global loads global compatibility - global _float_size + global _float_precision global _unpack_dispatch_table global xrange @@ -955,9 +975,9 @@ def __init(): # Auto-detect system float precision if sys.float_info.mant_dig == 53: - _float_size = 64 + _float_precision = "double" else: - _float_size = 32 + _float_precision = "single" # Map packb and unpackb to the appropriate version if sys.version_info[0] == 3: From 2f7b667ac45e0f277d01f8cd49e20e203991913e Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 18 Apr 2017 23:32:57 +0100 Subject: [PATCH 034/109] add tox config file and .tox/ to gitignore --- .gitignore | 1 + tox.ini | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index bf98a32..8ed73a2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dist/ *.egg-info/ *.pyc *.swp +.tox/ diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..321a63c --- /dev/null +++ b/tox.ini @@ -0,0 +1,6 @@ +[tox] +envlist = py27, py35, py36, pypy, pypy3 +skip_missing_interpreters=true +[testenv] +deps = pytest +commands = pytest From 456032c22dcdb54ce8f472cfc23018abe5360ef7 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 18 Apr 2017 23:33:30 +0100 Subject: [PATCH 035/109] change travis configuration to use tox to run tests --- .travis.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5932b92..8c1c587 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,17 @@ +sudo: false language: python -python: - - "2.7" - - "3.5" - - "pypy" - - "pypy3" -script: python test_umsgpack.py +install: pip install tox +script: tox + +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: 3.5 + env: TOXENV=py35 + - python: 3.6 + env: TOXENV=py36 + - python: pypy + env: TOXENV=pypy + - python: pypy3 + env: TOXENV=pypy3 From e60bc5e9352014f8b176b03f3e8c347545f34130 Mon Sep 17 00:00:00 2001 From: Pedro Rodrigues Date: Tue, 18 Apr 2017 23:36:43 +0100 Subject: [PATCH 036/109] add note about tox to README.md resolves #28. --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index e12c86a..3dab4eb 100644 --- a/README.md +++ b/README.md @@ -399,6 +399,13 @@ $ pypy test_umsgpack.py $ pypy3 test_umsgpack.py ``` +Alternatively, you can use `tox` or `detox` to test multiple Python versions at once. + +``` text +$ pip install tox +$ tox +``` + ## License u-msgpack-python is MIT licensed. See the included `LICENSE` file for more details. From cdf898069af690ea618fbb749f126b4315782e0e Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 19 Apr 2017 06:44:47 -0700 Subject: [PATCH 037/109] update version and changelog to v2.4.0 --- CHANGELOG.md | 10 ++++++++++ setup.py | 2 +- umsgpack.py | 6 +++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87ca470..6d8a30d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +* Version 2.4.0 - 04/20/2017 + * Add hash special method to Ext class. + * Add packing option to force floating point precision. + * Make codebase PEP 8 compliant. + * Add support for tox automated testing and use it in CI. + * Contributors + * Fabien Fleutot, @fab13n - 4c461ed, bdeee20 + * Yuhang (Steven) Wang, @yuhangwang - 5f53bcf + * Pedro Rodrigues, @medecau - 2f7b667, 456032c, e60bc5e + * Version 2.3.0 - 10/19/2016 * Add `ext_handlers` option to packing and unpacking functions to support application-defined Ext packing and unpacking hooks. * Add `allow_invalid_utf8` option to unpacking functions to allow unpacking of invalid UTF-8 strings. diff --git a/setup.py b/setup.py index b326345..68bcbdd 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.3.0', + version='2.4.0', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index 54da44c..cd3154b 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.3.0 - v at sergeev.io +# u-msgpack-python v2.4.0 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.3.0 - v at sergeev.io +u-msgpack-python v2.4.0 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -48,7 +48,7 @@ import sys import io -__version__ = "2.3.0" +__version__ = "2.4.0" "Module version string" version = (2, 3, 0) From 38c712520092128caa795762bb2c2fdf30275a06 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 25 Apr 2017 06:43:09 -0700 Subject: [PATCH 038/109] update version and changelog to v2.4.1 --- CHANGELOG.md | 3 +++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d8a30d..f516537 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +* Version 2.4.1 - 04/25/2017 + * Fix module version tuple inconsistency. + * Version 2.4.0 - 04/20/2017 * Add hash special method to Ext class. * Add packing option to force floating point precision. diff --git a/setup.py b/setup.py index 68bcbdd..e60d12d 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.4.0', + version='2.4.1', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index cd3154b..cd7a203 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.4.0 - v at sergeev.io +# u-msgpack-python v2.4.1 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.4.0 - v at sergeev.io +u-msgpack-python v2.4.1 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -48,10 +48,10 @@ import sys import io -__version__ = "2.4.0" +__version__ = "2.4.1" "Module version string" -version = (2, 3, 0) +version = (2, 4, 1) "Module version tuple" From 16510e9a716f8fc8f0c0daa87269dbf0c7d5ba1f Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Fri, 22 Sep 2017 12:09:30 +0100 Subject: [PATCH 039/109] fix tests for big-endian platforms On powerpc and powerpc64 one test fails as: ``` $ py.test -v ... test_umsgpack.py::TestUmsgpack::test_pack_ext_handler FAILED test_umsgpack.py::TestUmsgpack::test_unpack_ext_handler FAILED ... self = def test_pack_ext_handler(self): for (name, obj, data) in ext_handlers_test_vectors: obj_repr = repr(obj) print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) packed = umsgpack.packb(obj, ext_handlers=ext_handlers) > self.assertEqual(packed, data) E AssertionError: b'\xd7 ?\x80\x00\x00@\x00\x00\x00' != b'\xd7 \x00\x00\x80?\x00\x00\x00@' test_umsgpack.py:484: AssertionError ``` The problem here is in 'struct.pack' output: it uses native endianness format but test hardcodes little-endian output. The change forces 'struct.pack' into little-endian format. That way all tests pass:. Signed-off-by: Sergei Trofimovich Signed-off-by: Vanya A. Sergeev --- test_umsgpack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index a25b5b8..e500276 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -297,9 +297,9 @@ CustomType = namedtuple('CustomType', ['x', 'y', 'z']) ext_handlers = { - complex: lambda obj: umsgpack.Ext(0x20, struct.pack("ff", obj.real, obj.imag)), + complex: lambda obj: umsgpack.Ext(0x20, struct.pack(" Date: Wed, 26 Apr 2017 05:25:45 -0700 Subject: [PATCH 040/109] add tox .cache/ to gitgnore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8ed73a2..be2b6a6 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ *.pyc *.swp .tox/ +.cache/ From 70f3daf8df66b9f11bef3ca580616c7b11b89396 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 31 Mar 2018 19:54:44 -0500 Subject: [PATCH 041/109] add .pytest_cache/ to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index be2b6a6..2b225f4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ dist/ *.swp .tox/ .cache/ +.pytest_cache/ From 54d296fa9251b75e57cc5d9afde14501723e055d Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 26 Apr 2017 05:26:15 -0700 Subject: [PATCH 042/109] add support for msgpack timestamp format --- README.md | 54 +++++++++++++++++++------------ msgpack.org.md | 34 +++++++++----------- test_umsgpack.py | 28 +++++++++++++++- umsgpack.py | 83 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 158 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 3dab4eb..9f11bce 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # u-msgpack-python [![Build Status](https://travis-ci.org/vsergeev/u-msgpack-python.svg?branch=master)](https://travis-ci.org/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) -u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, and application-defined ext types. +u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. u-msgpack-python is currently distributed on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py). @@ -81,18 +81,18 @@ b'\x01\x02\x03' Serializing and deserializing application-defined types with Ext handlers: ``` python ->>> umsgpack.packb([complex(1,2), datetime.datetime.now()], +>>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], ... ext_handlers = { ... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), -... datetime.datetime: lambda obj: umsgpack.Ext(0x40, obj.strftime("%Y%m%dT%H:%M:%S.%f").encode()), -... }) -b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xc7\x18@20161017T00:12:53.719377' +... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xd6@0.31' >>> umsgpack.unpackb(_, ... ext_handlers = { ... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), -... 0x40: lambda ext: datetime.datetime.strptime(ext.data.decode(), "%Y%m%dT%H:%M:%S.%f"), -... }) -[(1+2j), datetime.datetime(2016, 10, 17, 0, 12, 53, 719377)] +... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), +... }) +[(1+2j), Decimal('0.31')] >>> ``` @@ -120,37 +120,35 @@ custom types to callables that pack the type into an Ext object. The callable should accept the custom type object as an argument and return a packed `umsgpack.Ext` object. -Example for packing `set`, `complex`, and `datetime.datetime` types into Ext +Example for packing `set`, `complex`, and `decimal.Decimal` types into Ext objects with type codes 0x20, 0x30, and 0x40, respectively: ``` python ->>> umsgpack.packb([1, True, {"foo", 2}, complex(3, 4), datetime.datetime.now()], +>>> umsgpack.packb([1, True, {"foo", 2}, complex(3, 4), decimal.Decimal("0.31")], ... ext_handlers = { ... set: lambda obj: umsgpack.Ext(0x20, umsgpack.packb(list(obj))), ... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), -... datetime.datetime: lambda obj: umsgpack.Ext(0x40, obj.strftime("%Y%m%dT%H:%M:%S.%f").encode()), -... }) -b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xc7\x18@20161015T02:28:35.666425' +... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31' >>> ``` - Similarly, the unpacking functions accept an optional `ext_handlers` dictionary that maps Ext type codes to callables that unpack the Ext into a custom object. The callable should accept a `umsgpack.Ext` object as an argument and return an unpacked custom type object. Example for unpacking Ext objects with type codes 0x20, 0x30, and 0x40, into -`set`, `complex`, and `datetime.datetime` typed objects, respectively: +`set`, `complex`, and `decimal.Decimal` typed objects, respectively: ``` python ->>> umsgpack.unpackb(b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@' \ -... b'\xc7\x18@20161015T02:28:35.666425', +>>> umsgpack.unpackb(b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31', ... ext_handlers = { ... 0x20: lambda ext: set(umsgpack.unpackb(ext.data)), ... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), -... 0x40: lambda ext: datetime.datetime.strptime(ext.data.decode(), "%Y%m%dT%H:%M:%S.%f"), -... }) -[1, True, {'foo', 2}, (3+4j), datetime.datetime(2016, 10, 15, 2, 28, 35, 666425)] +... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), +... }) +[1, True, {'foo', 2}, (3+4j), Decimal('0.31')] >>> ``` @@ -341,6 +339,20 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a >>> ``` +* `UnsupportedTimestampException`: Unsupported timestamp encountered during unpacking. + + The official timestamp extension type supports 32-bit, 64-bit and 96-bit + formats. This exception is thrown if a timestamp extension type with an + unsupported format is encountered. + + ``` python + # Attempt to unpack invalid timestamp + >>> umsgpack.unpackb(b"\xd5\xff\x01\x02") + ... + umsgpack.UnsupportedTimestampException: unsupported timestamp with data length 2 + >>> + ``` + * `ReservedCodeException`: Reserved code encountered during unpacking. ``` python @@ -387,6 +399,8 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * The msgpack array format is unpacked into a Python list, unless it is the key of a map, in which case it is unpacked into a Python tuple * Python tuples and lists are both packed into the msgpack array format * Python float types are packed into the msgpack float32 or float64 format depending on the system's `sys.float_info` +* The Python `datetime.datetime` type is packed into, and unpacked from, the msgpack `timestamp` format + * Note that this Python type only supports microsecond resolution, while the msgpack `timestamp` format supports nanosecond resolution. Timestamps with finer than microsecond resolution will lose precision during unpacking. ## Testing diff --git a/msgpack.org.md b/msgpack.org.md index c94d371..009e629 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -82,26 +82,22 @@ b'\x01\x02\x03' Serializing and deserializing application-defined types with Ext handlers: ``` python ->>> umsgpack.packb([complex(1,2), datetime.datetime.now()], -... ext_handlers = { -... complex: lambda obj: umsgpack.Ext(0x30, -... struct.pack("ff", obj.real, obj.imag)), -... datetime.datetime: lambda obj: umsgpack.Ext(0x40, -... obj.strftime("%Y%m%dT%H:%M:%S.%f").encode()), -... }) -b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xc7\x18@20161017T00:12:53.7' -b'19377' +>>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], +... ext_handlers = { +... complex: lambda obj: +... umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... decimal.Decimal: lambda obj: +... umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xd6@0.31' >>> umsgpack.unpackb(_, -... ext_handlers = { -... 0x30: lambda ext: -... complex(*struct.unpack("ff", ext.data)), -... 0x40: lambda ext: -... datetime.datetime.strptime( -... ext.data.decode(), -... "%Y%m%dT%H:%M:%S.%f" -... ), -... }) -[(1+2j), datetime.datetime(2016, 10, 17, 0, 12, 53, 719377)] +... ext_handlers = { +... 0x30: lambda ext: +... complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: +... decimal.Decimal(ext.data.decode()), +... }) +[(1+2j), Decimal('0.31')] >>> ``` diff --git a/test_umsgpack.py b/test_umsgpack.py index e500276..926e0c6 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -11,6 +11,7 @@ import sys import struct import unittest +import datetime import io from collections import OrderedDict, namedtuple @@ -116,6 +117,27 @@ ["empty array", [], b"\x90"], # Empty Map ["empty map", {}, b"\x80"], + # 32-bit Timestamp + ["32-bit timestamp", datetime.datetime(1970, 1, 1, 0, 0, 0, 0, umsgpack._utc_tzinfo), + b"\xd6\xff\x00\x00\x00\x00"], + ["32-bit timestamp", datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xd6\xff\x38\x6d\xd1\x4e"], + # 64-bit Timestamp + ["64-bit timestamp", datetime.datetime(2000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x4b\x51\x40\x38\x6d\xd1\x4e"], + ["64-bit timestamp", datetime.datetime(2200, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x00\x00\x01\xb0\x9e\xa6\xce"], + ["64-bit timestamp", datetime.datetime(2200, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x4b\x51\x41\xb0\x9e\xa6\xce"], + # 96-bit Timestamp + ["96-bit timestamp", datetime.datetime(1900, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x00\x00\x00\xff\xff\xff\xff\x7c\x56\x0f\x4e"], + ["96-bit timestamp", datetime.datetime(1900, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x12\xd4\x50\xff\xff\xff\xff\x7c\x56\x0f\x4e"], + ["96-bit timestamp", datetime.datetime(3000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x00\x00\x00\x00\x00\x00\x07\x91\x5f\x59\xce"], + ["96-bit timestamp", datetime.datetime(3000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x12\xd4\x50\x00\x00\x00\x07\x91\x5f\x59\xce"], ] composite_test_vectors = [ @@ -262,6 +284,9 @@ # Reserved code (0xc1) ["reserved code", b"\xc1", umsgpack.ReservedCodeException], + # Unsupported timestamp (unsupported data length) + ["unsupported timestamp", b"\xc7\x02\xff\xaa\xbb", + umsgpack.UnsupportedTimestampException], # Invalid string (non utf-8) ["invalid string", b"\xa1\x80", umsgpack.InvalidStringException], @@ -318,6 +343,7 @@ "UnsupportedTypeException", "InsufficientDataException", "InvalidStringException", + "UnsupportedTimestampException", "ReservedCodeException", "UnhashableKeyException", "DuplicateKeyException", @@ -519,7 +545,7 @@ def test_namespacing(self): exported_vars = list(filter(lambda x: not x.startswith("_"), dir(umsgpack))) # Ignore imports - exported_vars = list(filter(lambda x: x != "struct" and x != "collections" and x != + exported_vars = list(filter(lambda x: x != "struct" and x != "collections" and x != "datetime" and x != "sys" and x != "io" and x != "xrange", exported_vars)) self.assertTrue(len(exported_vars) == len(exported_vars_test_vector)) diff --git a/umsgpack.py b/umsgpack.py index cd7a203..7e2beb4 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -45,6 +45,7 @@ """ import struct import collections +import datetime import sys import io @@ -168,6 +169,11 @@ class InvalidStringException(UnpackException): pass +class UnsupportedTimestampException(UnpackException): + "Unsupported timestamp format encountered during unpacking." + pass + + class ReservedCodeException(UnpackException): "Reserved code encountered during unpacking." pass @@ -341,6 +347,29 @@ def _pack_ext(obj, fp, options): raise UnsupportedTypeException("huge ext data") +def _pack_ext_timestamp(obj, fp, options): + delta = obj - _epoch + seconds = delta.seconds + delta.days * 86400 + microseconds = delta.microseconds + + if microseconds == 0 and 0 <= seconds <= 2**32 - 1: + # 32-bit timestamp + fp.write(b"\xd6\xff" + + struct.pack(">I", seconds)) + elif 0 <= seconds <= 2**34 - 1: + # 64-bit timestamp + value = ((microseconds * 1000) << 34) | seconds + fp.write(b"\xd7\xff" + + struct.pack(">Q", value)) + elif -2**63 <= abs(seconds) <= 2**63 - 1: + # 96-bit timestamp + fp.write(b"\xc7\x0c\xff" + + struct.pack(">I", microseconds * 1000) + + struct.pack(">q", seconds)) + else: + raise UnsupportedTypeException("huge timestamp") + + def _pack_array(obj, fp, options): if len(obj) <= 15: fp.write(struct.pack("B", 0x90 | len(obj))) @@ -428,6 +457,8 @@ def _pack2(obj, fp, **options): _pack_array(obj, fp, options) elif isinstance(obj, dict): _pack_map(obj, fp, options) + elif isinstance(obj, datetime.datetime): + _pack_ext_timestamp(obj, fp, options) elif isinstance(obj, Ext): _pack_ext(obj, fp, options) elif ext_handlers: @@ -498,6 +529,8 @@ def _pack3(obj, fp, **options): _pack_array(obj, fp, options) elif isinstance(obj, dict): _pack_map(obj, fp, options) + elif isinstance(obj, datetime.datetime): + _pack_ext_timestamp(obj, fp, options) elif isinstance(obj, Ext): _pack_ext(obj, fp, options) elif ext_handlers: @@ -703,7 +736,15 @@ def _unpack_ext(code, fp, options): else: raise Exception("logic error, not ext: 0x%02x" % ord(code)) - ext = Ext(ord(_read_except(fp, 1)), _read_except(fp, length)) + ext_type = struct.unpack("b", _read_except(fp, 1))[0] + ext_data = _read_except(fp, length) + + # Timestamp extension + if ext_type == -1: + return _unpack_ext_timestamp(code, ext_data, options) + + # Application extension + ext = Ext(ext_type, ext_data) # Unpack with ext handler, if we have one ext_handlers = options.get("ext_handlers") @@ -713,6 +754,28 @@ def _unpack_ext(code, fp, options): return ext +def _unpack_ext_timestamp(code, data, options): + if len(data) == 4: + # 32-bit timestamp + seconds = struct.unpack(">I", data)[0] + microseconds = 0 + elif len(data) == 8: + # 64-bit timestamp + value = struct.unpack(">Q", data)[0] + seconds = value & 0x3ffffffff + microseconds = (value >> 34) // 1000 + elif len(data) == 12: + # 96-bit timestamp + seconds = struct.unpack(">q", data[4:12])[0] + microseconds = struct.unpack(">I", data[0:4])[0] // 1000 + else: + raise UnsupportedTimestampException( + "unsupported timestamp with data length %d" % len(data)) + + return _epoch + datetime.timedelta(seconds=seconds, + microseconds=microseconds) + + def _unpack_array(code, fp, options): if (ord(code) & 0xf0) == 0x90: length = (ord(code) & ~0xf0) @@ -801,6 +864,8 @@ def _unpack2(fp, **options): Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. ReservedCodeException(UnpackException): Reserved code encountered during unpacking. UnhashableKeyException(UnpackException): @@ -843,6 +908,8 @@ def _unpack3(fp, **options): Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. ReservedCodeException(UnpackException): Reserved code encountered during unpacking. UnhashableKeyException(UnpackException): @@ -888,6 +955,8 @@ def _unpackb2(s, **options): Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. ReservedCodeException(UnpackException): Reserved code encountered during unpacking. UnhashableKeyException(UnpackException): @@ -934,6 +1003,8 @@ def _unpackb3(s, **options): Insufficient data to unpack the serialized object. InvalidStringException(UnpackException): Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. ReservedCodeException(UnpackException): Reserved code encountered during unpacking. UnhashableKeyException(UnpackException): @@ -966,6 +1037,8 @@ def __init(): global load global loads global compatibility + global _epoch + global _utc_tzinfo global _float_precision global _unpack_dispatch_table global xrange @@ -973,6 +1046,14 @@ def __init(): # Compatibility mode for handling strings/bytes with the old specification compatibility = False + if sys.version_info[0] == 3: + _utc_tzinfo = datetime.timezone.utc + else: + _utc_tzinfo = None + + # Calculate epoch datetime + _epoch = datetime.datetime(1970, 1, 1, tzinfo=_utc_tzinfo) + # Auto-detect system float precision if sys.float_info.mant_dig == 53: _float_precision = "double" From f035453d257ac45c4459feda1453ed08924889a1 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 28 Apr 2017 05:23:00 -0700 Subject: [PATCH 043/109] allow user override of reserved msgpack ext types --- README.md | 2 +- test_umsgpack.py | 41 +++++++++++++++++++++++++++++++++++++---- umsgpack.py | 42 +++++++++++++++++++----------------------- 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 9f11bce..3cd9ee1 100644 --- a/README.md +++ b/README.md @@ -400,7 +400,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * Python tuples and lists are both packed into the msgpack array format * Python float types are packed into the msgpack float32 or float64 format depending on the system's `sys.float_info` * The Python `datetime.datetime` type is packed into, and unpacked from, the msgpack `timestamp` format - * Note that this Python type only supports microsecond resolution, while the msgpack `timestamp` format supports nanosecond resolution. Timestamps with finer than microsecond resolution will lose precision during unpacking. + * Note that this Python type only supports microsecond resolution, while the msgpack `timestamp` format supports nanosecond resolution. Timestamps with finer than microsecond resolution will lose precision during unpacking. Users may override the packing and unpacking of the msgpack `timestamp` format with a custom type for alternate behavior. ## Testing diff --git a/test_umsgpack.py b/test_umsgpack.py index 926e0c6..eeea33e 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -334,6 +334,22 @@ b"\xd7\x30\x93\xc4\x03\x61\x62\x63\x7b\xc3"], ] +override_ext_handlers = { + datetime.datetime: + lambda obj: umsgpack.Ext(0x40, obj.strftime("%Y%m%dT%H:%M:%S.%f").encode()), + -0x01: + lambda ext: ext, +} + +override_ext_handlers_test_vectors = [ + ["pack override", + datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b'\xc7\x18@20000101T10:05:02.000000'], + ["unpack override", + umsgpack.Ext(-0x01, b"\x00\xbb\xcc\xdd\x01\x02\x03\x04\x05\x06\x07\x08"), + b'\xc7\x0c\xff\x00\xbb\xcc\xdd\x01\x02\x03\x04\x05\x06\x07\x08'], +] + # These are the only global variables that should be exported by umsgpack exported_vars_test_vector = [ "Ext", @@ -492,10 +508,7 @@ def test_unpack_ordered_dict(self): def test_ext_exceptions(self): with self.assertRaises(TypeError): - _ = umsgpack.Ext(-1, b"") - - with self.assertRaises(TypeError): - _ = umsgpack.Ext(128, b"") + _ = umsgpack.Ext(5.0, b"") with self.assertRaises(TypeError): _ = umsgpack.Ext(0, u"unicode string") @@ -527,6 +540,26 @@ def test_pack_force_float_precision(self): packed = umsgpack.packb(obj, force_float_precision=precision) self.assertEqual(packed, data) + def test_pack_ext_override(self): + # Test overridden packing of datetime.datetime + (name, obj, data) = override_ext_handlers_test_vectors[0] + obj_repr = repr(obj) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj, ext_handlers=override_ext_handlers) + self.assertEqual(packed, data) + + def test_unpack_ext_override(self): + # Test overridden unpacking of Ext type -1 + (name, obj, data) = override_ext_handlers_test_vectors[1] + obj_repr = repr(obj) + print("\tTesting %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + unpacked = umsgpack.unpackb(data, ext_handlers=override_ext_handlers) + self.assertEqual(unpacked, obj) + def test_streaming_writer(self): # Try first composite test vector (_, obj, data) = composite_test_vectors[0] diff --git a/umsgpack.py b/umsgpack.py index 7e2beb4..e7ff7a6 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -72,13 +72,9 @@ def __init__(self, type, data): Construct a new Ext object. Args: - type: application-defined type integer from 0 to 127 + type: application-defined type integer data: application-defined data byte array - Raises: - TypeError: - Specified ext type is outside of 0 to 127 range. - Example: >>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) @@ -88,9 +84,9 @@ def __init__(self, type, data): Ext Object (Type: 0x05, Data: 01 02 03) >>> """ - # Application ext type should be 0 <= type <= 127 - if not isinstance(type, int) or not (type >= 0 and type <= 127): - raise TypeError("ext type out of range") + # Check type is type int + if not isinstance(type, int): + raise TypeError("ext type is not type integer") # Check data is type bytes elif sys.version_info[0] == 3 and not isinstance(data, bytes): raise TypeError("ext data is not type \'bytes\'") @@ -739,38 +735,38 @@ def _unpack_ext(code, fp, options): ext_type = struct.unpack("b", _read_except(fp, 1))[0] ext_data = _read_except(fp, length) - # Timestamp extension - if ext_type == -1: - return _unpack_ext_timestamp(code, ext_data, options) - - # Application extension + # Create extension object ext = Ext(ext_type, ext_data) # Unpack with ext handler, if we have one ext_handlers = options.get("ext_handlers") if ext_handlers and ext.type in ext_handlers: - ext = ext_handlers[ext.type](ext) + return ext_handlers[ext.type](ext) + + # Timestamp extension + if ext.type == -1: + return _unpack_ext_timestamp(ext, options) return ext -def _unpack_ext_timestamp(code, data, options): - if len(data) == 4: +def _unpack_ext_timestamp(ext, options): + if len(ext.data) == 4: # 32-bit timestamp - seconds = struct.unpack(">I", data)[0] + seconds = struct.unpack(">I", ext.data)[0] microseconds = 0 - elif len(data) == 8: + elif len(ext.data) == 8: # 64-bit timestamp - value = struct.unpack(">Q", data)[0] + value = struct.unpack(">Q", ext.data)[0] seconds = value & 0x3ffffffff microseconds = (value >> 34) // 1000 - elif len(data) == 12: + elif len(ext.data) == 12: # 96-bit timestamp - seconds = struct.unpack(">q", data[4:12])[0] - microseconds = struct.unpack(">I", data[0:4])[0] // 1000 + seconds = struct.unpack(">q", ext.data[4:12])[0] + microseconds = struct.unpack(">I", ext.data[0:4])[0] // 1000 else: raise UnsupportedTimestampException( - "unsupported timestamp with data length %d" % len(data)) + "unsupported timestamp with data length %d" % len(ext.data)) return _epoch + datetime.timedelta(seconds=seconds, microseconds=microseconds) From 5665042f6c5d593219233d116fdba2b1e6ffb14f Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 31 Mar 2018 19:55:05 -0500 Subject: [PATCH 044/109] update version and changelog to v2.5.0 --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f516537..a9b8280 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +* Version 2.5.0 - 03/31/2018 + * Add support for the timestamp extension type. + * Fix tests on big endian platforms + * Contributors + * Sergei Trofimovich, @trofi - 16510e9 + * Version 2.4.1 - 04/25/2017 * Fix module version tuple inconsistency. diff --git a/setup.py b/setup.py index e60d12d..84a1223 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.4.1', + version='2.5.0', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index e7ff7a6..139ab98 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.4.1 - v at sergeev.io +# u-msgpack-python v2.5.0 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.4.1 - v at sergeev.io +u-msgpack-python v2.5.0 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -49,10 +49,10 @@ import sys import io -__version__ = "2.4.1" +__version__ = "2.5.0" "Module version string" -version = (2, 4, 1) +version = (2, 5, 0) "Module version tuple" From 28907ba12f387d952c3930412a816ef1605cb334 Mon Sep 17 00:00:00 2001 From: Gabe Appleton Date: Fri, 22 Jun 2018 23:23:28 -0700 Subject: [PATCH 045/109] make Ext a new-style object for cleaner python2 inheritance to fix errors when user defined types inherit from Ext in Python 2. Signed-off-by: Vanya A. Sergeev --- umsgpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/umsgpack.py b/umsgpack.py index 139ab98..698f483 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -61,7 +61,7 @@ ############################################################################## # Extension type for application-defined types and data -class Ext: +class Ext(object): """ The Ext class facilitates creating a serializable extension object to store an application-defined type and data byte array. From 76751f334dfb17ebdf1b61c1174922dc0ecca252 Mon Sep 17 00:00:00 2001 From: Gabe Appleton Date: Wed, 25 Jul 2018 14:13:22 -0700 Subject: [PATCH 046/109] reduce calls to len() and favor < to <= in comparisons for minor performance improvement. Signed-off-by: Vanya A. Sergeev --- umsgpack.py | 109 ++++++++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 51 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index 698f483..43df2f4 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -250,15 +250,15 @@ def _pack_integer(obj, fp, options): else: raise UnsupportedTypeException("huge signed int") else: - if obj <= 127: + if obj < 128: fp.write(struct.pack("B", obj)) - elif obj <= 2**8 - 1: + elif obj < 2**8: fp.write(b"\xcc" + struct.pack("B", obj)) - elif obj <= 2**16 - 1: + elif obj < 2**16: fp.write(b"\xcd" + struct.pack(">H", obj)) - elif obj <= 2**32 - 1: + elif obj < 2**32: fp.write(b"\xce" + struct.pack(">I", obj)) - elif obj <= 2**64 - 1: + elif obj < 2**64: fp.write(b"\xcf" + struct.pack(">Q", obj)) else: raise UnsupportedTypeException("huge unsigned int") @@ -285,60 +285,64 @@ def _pack_float(obj, fp, options): def _pack_string(obj, fp, options): obj = obj.encode('utf-8') - if len(obj) <= 31: - fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**8 - 1: - fp.write(b"\xd9" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) + obj_len = len(obj) + if obj_len < 32: + fp.write(struct.pack("B", 0xa0 | obj_len) + obj) + elif obj_len < 2**8: + fp.write(b"\xd9" + struct.pack("B", obj_len) + obj) + elif obj_len < 2**16: + fp.write(b"\xda" + struct.pack(">H", obj_len) + obj) + elif obj_len < 2**32: + fp.write(b"\xdb" + struct.pack(">I", obj_len) + obj) else: raise UnsupportedTypeException("huge string") def _pack_binary(obj, fp, options): - if len(obj) <= 2**8 - 1: - fp.write(b"\xc4" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xc5" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xc6" + struct.pack(">I", len(obj)) + obj) + obj_len = len(obj) + if obj_len < 2**8: + fp.write(b"\xc4" + struct.pack("B", obj_len) + obj) + elif obj_len < 2**16: + fp.write(b"\xc5" + struct.pack(">H", obj_len) + obj) + elif obj_len < 2**32: + fp.write(b"\xc6" + struct.pack(">I", obj_len) + obj) else: raise UnsupportedTypeException("huge binary string") def _pack_oldspec_raw(obj, fp, options): - if len(obj) <= 31: - fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) + obj_len = len(obj) + if obj_len < 32: + fp.write(struct.pack("B", 0xa0 | obj_len) + obj) + elif obj_len < 2**16: + fp.write(b"\xda" + struct.pack(">H", obj_len) + obj) + elif obj_len < 2**32: + fp.write(b"\xdb" + struct.pack(">I", obj_len) + obj) else: raise UnsupportedTypeException("huge raw string") def _pack_ext(obj, fp, options): - if len(obj.data) == 1: + obj_len = len(obj.data) + if obj_len == 1: fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 2: + elif obj_len == 2: fp.write(b"\xd5" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 4: + elif obj_len == 4: fp.write(b"\xd6" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 8: + elif obj_len == 8: fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 16: + elif obj_len == 16: fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**8 - 1: + elif obj_len < 2**8: fp.write(b"\xc7" + - struct.pack("BB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**16 - 1: + struct.pack("BB", obj_len, obj.type & 0xff) + obj.data) + elif obj_len < 2**16: fp.write(b"\xc8" + - struct.pack(">HB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**32 - 1: + struct.pack(">HB", obj_len, obj.type & 0xff) + obj.data) + elif obj_len < 2**32: fp.write(b"\xc9" + - struct.pack(">IB", len(obj.data), obj.type & 0xff) + obj.data) + struct.pack(">IB", obj_len, obj.type & 0xff) + obj.data) else: raise UnsupportedTypeException("huge ext data") @@ -367,12 +371,13 @@ def _pack_ext_timestamp(obj, fp, options): def _pack_array(obj, fp, options): - if len(obj) <= 15: - fp.write(struct.pack("B", 0x90 | len(obj))) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xdc" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdd" + struct.pack(">I", len(obj))) + obj_len = len(obj) + if obj_len < 16: + fp.write(struct.pack("B", 0x90 | obj_len)) + elif obj_len < 2**16: + fp.write(b"\xdc" + struct.pack(">H", obj_len)) + elif obj_len < 2**32: + fp.write(b"\xdd" + struct.pack(">I", obj_len)) else: raise UnsupportedTypeException("huge array") @@ -381,12 +386,13 @@ def _pack_array(obj, fp, options): def _pack_map(obj, fp, options): - if len(obj) <= 15: - fp.write(struct.pack("B", 0x80 | len(obj))) - elif len(obj) <= 2**16 - 1: - fp.write(b"\xde" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32 - 1: - fp.write(b"\xdf" + struct.pack(">I", len(obj))) + obj_len = len(obj) + if obj_len < 16: + fp.write(struct.pack("B", 0x80 | obj_len)) + elif obj_len < 2**16: + fp.write(b"\xde" + struct.pack(">H", obj_len)) + elif obj_len < 2**32: + fp.write(b"\xdf" + struct.pack(">I", obj_len)) else: raise UnsupportedTypeException("huge array") @@ -751,16 +757,17 @@ def _unpack_ext(code, fp, options): def _unpack_ext_timestamp(ext, options): - if len(ext.data) == 4: + obj_len = len(ext.data) + if obj_len == 4: # 32-bit timestamp seconds = struct.unpack(">I", ext.data)[0] microseconds = 0 - elif len(ext.data) == 8: + elif obj_len == 8: # 64-bit timestamp value = struct.unpack(">Q", ext.data)[0] seconds = value & 0x3ffffffff microseconds = (value >> 34) // 1000 - elif len(ext.data) == 12: + elif obj_len == 12: # 96-bit timestamp seconds = struct.unpack(">q", ext.data[4:12])[0] microseconds = struct.unpack(">I", ext.data[0:4])[0] // 1000 From 7a9b7176bb5bbd0e29db09f92f9b32dc9a7774fd Mon Sep 17 00:00:00 2001 From: Gabe Appleton Date: Mon, 20 Aug 2018 14:22:12 -0700 Subject: [PATCH 047/109] use tuple in isinstance() to simplify a few instance checks Turn `isinstance(a, b) or isinstance(a, c)` into `isinstance(a, (b, c))` for minor performance improvement. Signed-off-by: Vanya A. Sergeev --- umsgpack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index 43df2f4..372455b 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -443,7 +443,7 @@ def _pack2(obj, fp, **options): _pack_ext(ext_handlers[obj.__class__](obj), fp, options) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) - elif isinstance(obj, int) or isinstance(obj, long): + elif isinstance(obj, (int, long)): _pack_integer(obj, fp, options) elif isinstance(obj, float): _pack_float(obj, fp, options) @@ -455,7 +455,7 @@ def _pack2(obj, fp, **options): _pack_string(obj, fp, options) elif isinstance(obj, str): _pack_binary(obj, fp, options) - elif isinstance(obj, list) or isinstance(obj, tuple): + elif isinstance(obj, (list, tuple)): _pack_array(obj, fp, options) elif isinstance(obj, dict): _pack_map(obj, fp, options) @@ -527,7 +527,7 @@ def _pack3(obj, fp, **options): _pack_string(obj, fp, options) elif isinstance(obj, bytes): _pack_binary(obj, fp, options) - elif isinstance(obj, list) or isinstance(obj, tuple): + elif isinstance(obj, (list, tuple)): _pack_array(obj, fp, options) elif isinstance(obj, dict): _pack_map(obj, fp, options) From 9ad593edb60db87388918c172536e4330db7f5e2 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 4 Sep 2018 06:03:02 -0500 Subject: [PATCH 048/109] add py37 environment to tox config --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 321a63c..7b9c7d0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, pypy, pypy3 +envlist = py27, py35, py36, py37, pypy, pypy3 skip_missing_interpreters=true [testenv] deps = pytest From 1a9e6066b773968e50e98b5670b01b1499923090 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 4 Sep 2018 06:15:46 -0500 Subject: [PATCH 049/109] add commented py37 tox environment to travis config until travis-ci/travis-ci#9815 is resolved. --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8c1c587..aeaf634 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,8 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 +# - python: 3.7 +# env: TOXENV=py37 - python: pypy env: TOXENV=pypy - python: pypy3 From 64dfc387d31fd293a2fc929fd3a80b91f62b7a49 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 3 Mar 2019 19:07:49 -0600 Subject: [PATCH 050/109] fix naive/aware timestamp handling in packing fixes #34. --- test_umsgpack.py | 30 ++++++++++++++++++++++++++++++ umsgpack.py | 25 ++++++++++++++++++++++--- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index eeea33e..42a6748 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -319,6 +319,18 @@ ["float precision double", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00"], ] +naive_timestamp_test_vectors = [ + ["32-bit timestamp (naive)", datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xd6\xff\x38\x6d\xd1\x4e", + datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo)], + ["64-bit timestamp (naive)", datetime.datetime(2200, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x4b\x51\x41\xb0\x9e\xa6\xce", + datetime.datetime(2200, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo)], + ["96-bit timestamp (naive)", datetime.datetime(3000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x12\xd4\x50\x00\x00\x00\x07\x91\x5f\x59\xce", + datetime.datetime(3000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo)], +] + CustomType = namedtuple('CustomType', ['x', 'y', 'z']) ext_handlers = { @@ -540,6 +552,24 @@ def test_pack_force_float_precision(self): packed = umsgpack.packb(obj, force_float_precision=precision) self.assertEqual(packed, data) + def test_pack_naive_timestamp(self): + for (name, obj, data, _) in naive_timestamp_test_vectors: + obj_repr = repr(obj) + print("\t Testing %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj) + self.assertEqual(packed, data) + + def test_unpack_naive_timestamp(self): + for (name, _, data, obj) in naive_timestamp_test_vectors: + obj_repr = repr(obj) + print("\t Testing %s: object %s" % + (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + unpacked = umsgpack.unpackb(data) + self.assertEqual(unpacked, obj) + def test_pack_ext_override(self): # Test overridden packing of datetime.datetime (name, obj, data) = override_ext_handlers_test_vectors[0] diff --git a/umsgpack.py b/umsgpack.py index 372455b..bb0d6c8 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -348,7 +348,14 @@ def _pack_ext(obj, fp, options): def _pack_ext_timestamp(obj, fp, options): - delta = obj - _epoch + if not obj.tzinfo: + # Object is naive datetime, convert to aware date time, + # assuming UTC timezone + delta = obj.replace(tzinfo=_utc_tzinfo) - _epoch + else: + # Object is aware datetime + delta = obj - _epoch + seconds = delta.seconds + delta.days * 86400 microseconds = delta.microseconds @@ -1052,9 +1059,21 @@ def __init(): if sys.version_info[0] == 3: _utc_tzinfo = datetime.timezone.utc else: - _utc_tzinfo = None + class UTC(datetime.tzinfo): + ZERO = datetime.timedelta(0) + + def utcoffset(self, dt): + return UTC.ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return UTC.ZERO + + _utc_tzinfo = UTC() - # Calculate epoch datetime + # Calculate an aware epoch datetime _epoch = datetime.datetime(1970, 1, 1, tzinfo=_utc_tzinfo) # Auto-detect system float precision From ec7077ee70e9f9e9470a88f49693d1376b136ee7 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 3 Mar 2019 19:12:33 -0600 Subject: [PATCH 051/109] add note about naive/aware timestamps to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3cd9ee1..20360ed 100644 --- a/README.md +++ b/README.md @@ -401,6 +401,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * Python float types are packed into the msgpack float32 or float64 format depending on the system's `sys.float_info` * The Python `datetime.datetime` type is packed into, and unpacked from, the msgpack `timestamp` format * Note that this Python type only supports microsecond resolution, while the msgpack `timestamp` format supports nanosecond resolution. Timestamps with finer than microsecond resolution will lose precision during unpacking. Users may override the packing and unpacking of the msgpack `timestamp` format with a custom type for alternate behavior. + * Both naive and aware timestamp are supported. Naive timestamps are packed as if they are in the UTC timezone. Timestamps are always unpacked as aware `datetime.datetime` objects in the UTC timezone. ## Testing From 8c2d6093068c7a23176099e046879ba5dd297402 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 3 Mar 2019 19:14:44 -0600 Subject: [PATCH 052/109] enable py37 tox environment in travis config --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index aeaf634..8e216ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,5 @@ sudo: false +dist: xenial language: python install: pip install tox script: tox @@ -11,8 +12,8 @@ matrix: env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 -# - python: 3.7 -# env: TOXENV=py37 + - python: 3.7 + env: TOXENV=py37 - python: pypy env: TOXENV=pypy - python: pypy3 From 49fa885c2fd247576546b16ac295735e271d5763 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 3 Mar 2019 19:55:59 -0600 Subject: [PATCH 053/109] update pypy version names in travis config for xenial --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e216ef..626748d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ matrix: env: TOXENV=py36 - python: 3.7 env: TOXENV=py37 - - python: pypy + - python: pypy2.7-6.0 env: TOXENV=pypy - - python: pypy3 + - python: pypy3.5-6.0 env: TOXENV=pypy3 From 50b1dd344e2962f4e7f56f6467310a0d0232bdbe Mon Sep 17 00:00:00 2001 From: DisposaBoy Date: Fri, 28 Sep 2018 10:12:51 +0100 Subject: [PATCH 054/109] add handling for short reads in file unpacking fixes #39. Signed-off-by: Vanya A. Sergeev --- test_umsgpack.py | 24 ++++++++++++++++++++++++ umsgpack.py | 13 ++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index 42a6748..851b059 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -615,6 +615,30 @@ def test_namespacing(self): for var in exported_vars_test_vector: self.assertTrue(var in exported_vars) + def test_load_short_read(self): + # When reading from files, the network, etc. there's no guarantee that + # read(n) returns n bytes. Simulate this with a file-like object that + # returns 1 byte at a time. + + class SlowFile(object): + def __init__(self, data): + self._data = data + + def read(self, n=None): + if n is None or len(self._data) == 0: + data, self._data = self._data, b'' + return data + + chunk = self._data[0:1] + self._data = self._data[1:] + return chunk + + obj = {'hello': 'world'} + f = SlowFile(umsgpack.dumps(obj)) + unpacked = umsgpack.load(f) + + self.assertEqual(unpacked, obj) + if __name__ == '__main__': unittest.main() diff --git a/umsgpack.py b/umsgpack.py index bb0d6c8..09f2b29 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -626,9 +626,20 @@ def _packb3(obj, **options): def _read_except(fp, n): + if n == 0: + return b"" + data = fp.read(n) - if len(data) < n: + if len(data) == 0: raise InsufficientDataException() + + while len(data) < n: + chunk = fp.read(n - len(data)) + if len(chunk) == 0: + raise InsufficientDataException() + + data += chunk + return data From e290b768ce63177ae04ed96402915b75a9741f38 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 3 Mar 2019 19:43:12 -0600 Subject: [PATCH 055/109] update version and changelog to v2.5.1 --- CHANGELOG.md | 9 +++++++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9b8280..09b72de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +* Version 2.5.1 - 03/03/2019 + * Fix handling of naive/aware datetime objects when packing the timestamp extension type. + * Add handling for short reads during file object unpacking. + * Make Ext base class a new-style object for cleaner inheritance in Python 2. + * Improve length comparisons and instance checks for minor performance improvement. + * Contributors + * Gabe Appleton, @gappleto97 - 28907ba, 76751f3, 7a9b717 + * DisposaBoy, @DisposaBoy - 50b1dd3 + * Version 2.5.0 - 03/31/2018 * Add support for the timestamp extension type. * Fix tests on big endian platforms diff --git a/setup.py b/setup.py index 84a1223..4173c8e 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.5.0', + version='2.5.1', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index 09f2b29..deafcc7 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.5.0 - v at sergeev.io +# u-msgpack-python v2.5.1 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.5.0 - v at sergeev.io +u-msgpack-python v2.5.1 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -49,10 +49,10 @@ import sys import io -__version__ = "2.5.0" +__version__ = "2.5.1" "Module version string" -version = (2, 5, 0) +version = (2, 5, 1) "Module version tuple" From 5ece62affdd47ac28a5737119653ff861d8008ee Mon Sep 17 00:00:00 2001 From: Gabe Appleton Date: Thu, 18 Apr 2019 15:56:05 -0700 Subject: [PATCH 056/109] fix DeprecationWarning about using ABCs from collections on Python 3.7 --- test_umsgpack.py | 2 +- umsgpack.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index 851b059..86d4362 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -609,7 +609,7 @@ def test_namespacing(self): dir(umsgpack))) # Ignore imports exported_vars = list(filter(lambda x: x != "struct" and x != "collections" and x != "datetime" and x != - "sys" and x != "io" and x != "xrange", exported_vars)) + "sys" and x != "io" and x != "xrange" and x != "Hashable", exported_vars)) self.assertTrue(len(exported_vars) == len(exported_vars_test_vector)) for var in exported_vars_test_vector: diff --git a/umsgpack.py b/umsgpack.py index deafcc7..ac37413 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -49,6 +49,11 @@ import sys import io +if sys.version_info[0:2] >= (3, 3): + from collections.abc import Hashable +else: + from collections import Hashable + __version__ = "2.5.1" "Module version string" @@ -835,7 +840,7 @@ def _unpack_map(code, fp, options): if isinstance(k, list): # Attempt to convert list into a hashable tuple k = _deep_list_to_tuple(k) - elif not isinstance(k, collections.Hashable): + elif not isinstance(k, Hashable): raise UnhashableKeyException( "encountered unhashable key: %s, %s" % (str(k), str(type(k)))) elif k in d: From 027bc3f9f45de75d570ff9239677d8b92e7fd7a1 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Thu, 15 Aug 2019 02:18:16 -0500 Subject: [PATCH 057/109] update version and changelog to v2.5.2 --- CHANGELOG.md | 5 +++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 09b72de..ad392b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +* Version 2.5.2 - 08/15/2019 + * Fix DeprecationWarning about using ABCs from 'collections' on Python 3.7. + * Contributors + * Gabe Appleton, @gappleto97 - 5ece62a + * Version 2.5.1 - 03/03/2019 * Fix handling of naive/aware datetime objects when packing the timestamp extension type. * Add handling for short reads during file object unpacking. diff --git a/setup.py b/setup.py index 4173c8e..d4230a2 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.5.1', + version='2.5.2', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index ac37413..be21e7c 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.5.1 - v at sergeev.io +# u-msgpack-python v2.5.2 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.5.1 - v at sergeev.io +u-msgpack-python v2.5.2 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -54,10 +54,10 @@ else: from collections import Hashable -__version__ = "2.5.1" +__version__ = "2.5.2" "Module version string" -version = (2, 5, 1) +version = (2, 5, 2) "Module version tuple" From 6d514f9d0c54c475f6f58ed95ddf605c798171a5 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 1 Mar 2020 17:04:12 -0600 Subject: [PATCH 058/109] add use_tuple unpacking option to unpack arrays into tuples resolves #43. --- test_umsgpack.py | 19 +++++++++++++++++++ umsgpack.py | 11 +++++++++++ 2 files changed, 30 insertions(+) diff --git a/test_umsgpack.py b/test_umsgpack.py index 86d4362..31c01e1 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -319,6 +319,12 @@ ["float precision double", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00"], ] +tuple_test_vectors = [ + ["nested array", [0x01, [b"\x80", [[u"a", u"b", u"c"], True]]], + b"\x92\x01\x92\xc4\x01\x80\x92\x93\xa1a\xa1b\xa1c\xc3", + (0x01, (b"\x80", ((u"a", u"b", u"c"), True)))], +] + naive_timestamp_test_vectors = [ ["32-bit timestamp (naive)", datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), b"\xd6\xff\x38\x6d\xd1\x4e", @@ -518,6 +524,19 @@ def test_unpack_ordered_dict(self): self.assertTrue(isinstance(unpacked, OrderedDict)) self.assertEqual(unpacked, obj) + def test_unpack_tuple(self): + # Use tuple test vector + (_, obj, data, obj_tuple) = tuple_test_vectors[0] + + # Unpack with default options (list) + self.assertEqual(umsgpack.unpackb(data), obj) + + # Unpack with use_tuple=False (list) + self.assertEqual(umsgpack.unpackb(data, use_tuple=False), obj) + + # Unpack with use_tuple=True (tuple) + self.assertEqual(umsgpack.unpackb(data, use_tuple=True), obj_tuple) + def test_ext_exceptions(self): with self.assertRaises(TypeError): _ = umsgpack.Ext(5.0, b"") diff --git a/umsgpack.py b/umsgpack.py index be21e7c..ec11cc6 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -812,6 +812,9 @@ def _unpack_array(code, fp, options): else: raise Exception("logic error, not array: 0x%02x" % ord(code)) + if options.get('use_tuple'): + return tuple((_unpack(fp, options) for i in xrange(length))) + return [_unpack(fp, options) for i in xrange(length)] @@ -878,6 +881,8 @@ def _unpack2(fp, **options): Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) allow_invalid_utf8 (bool): unpack invalid strings into instances of InvalidString, for access to the bytes (default False) @@ -922,6 +927,8 @@ def _unpack3(fp, **options): Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) allow_invalid_utf8 (bool): unpack invalid strings into instances of InvalidString, for access to the bytes (default False) @@ -967,6 +974,8 @@ def _unpackb2(s, **options): Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) allow_invalid_utf8 (bool): unpack invalid strings into instances of InvalidString, for access to the bytes (default False) @@ -1015,6 +1024,8 @@ def _unpackb3(s, **options): Ext into an object use_ordered_dict (bool): unpack maps into OrderedDict, instead of unordered dict (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) allow_invalid_utf8 (bool): unpack invalid strings into instances of InvalidString, for access to the bytes (default False) From 4a3dabd6386875fe7d3ebe99cde506ad3a016521 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 1 Mar 2020 17:05:46 -0600 Subject: [PATCH 059/109] add use_tuple unpacking option usage to readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 20360ed..869f76e 100644 --- a/README.md +++ b/README.md @@ -225,6 +225,18 @@ OrderedDict([('compact', True), ('schema', 0)]) >>> ``` +## Tuples + +The unpacking functions provide a `use_tuple` option to unpack MessagePack arrays into tuples, rather than lists. + +``` python +>>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03') +['a', True, [1, [2, 3]]] +>>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03', use_tuple=True) +('a', True, (1, (2, 3))) +>>> +``` + ### Invalid UTF-8 Strings The unpacking functions provide an `allow_invalid_utf8` option to unpack MessagePack strings with invalid UTF-8 into the `umsgpack.InvalidString` type, instead of throwing an exception. The `umsgpack.InvalidString` type is a subclass of `bytes`, and can be used like any other `bytes` object. From 8c2ea2567c407ee99197d114263b8ad47521908b Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 1 Mar 2020 17:06:21 -0600 Subject: [PATCH 060/109] add py38 environment to tox config --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7b9c7d0..73d8b30 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, py37, pypy, pypy3 +envlist = py27, py35, py36, py37, py38, pypy, pypy3 skip_missing_interpreters=true [testenv] deps = pytest From 022ca7dc0848186c5a4084eebc13d8fa3d492575 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 1 Mar 2020 17:06:45 -0600 Subject: [PATCH 061/109] enable py38 tox environment in travis config --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 626748d..0b6e534 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ matrix: env: TOXENV=py36 - python: 3.7 env: TOXENV=py37 + - python: 3.8 + env: TOXENV=py38 - python: pypy2.7-6.0 env: TOXENV=pypy - python: pypy3.5-6.0 From 0e539c42c861d940abb896de357fbeb3ca42918c Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 24 Apr 2020 00:24:49 -0500 Subject: [PATCH 062/109] remove unnecessary pass statement from wrapper classes --- umsgpack.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index ec11cc6..78c6c21 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -135,7 +135,6 @@ def __hash__(self): class InvalidString(bytes): """Subclass of bytes to hold invalid UTF-8 strings.""" - pass ############################################################################## # Exceptions @@ -145,39 +144,32 @@ class InvalidString(bytes): # Base Exception classes class PackException(Exception): "Base class for exceptions encountered during packing." - pass class UnpackException(Exception): "Base class for exceptions encountered during unpacking." - pass # Packing error class UnsupportedTypeException(PackException): "Object type not supported for packing." - pass # Unpacking error class InsufficientDataException(UnpackException): "Insufficient data to unpack the serialized object." - pass class InvalidStringException(UnpackException): "Invalid UTF-8 string encountered during unpacking." - pass class UnsupportedTimestampException(UnpackException): "Unsupported timestamp format encountered during unpacking." - pass class ReservedCodeException(UnpackException): "Reserved code encountered during unpacking." - pass class UnhashableKeyException(UnpackException): @@ -185,12 +177,10 @@ class UnhashableKeyException(UnpackException): Unhashable key encountered during map unpacking. The serialized map cannot be deserialized into a Python dictionary. """ - pass class DuplicateKeyException(UnpackException): "Duplicate key encountered during map unpacking." - pass # Backwards compatibility From 9a1eabf5ba9798dad7ce445d34d84a0e6b4c9744 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 24 Apr 2020 00:28:35 -0500 Subject: [PATCH 063/109] improve line break formatting and maintain a max line length of 100. --- umsgpack.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index 78c6c21..a4cf00b 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -104,9 +104,8 @@ def __eq__(self, other): """ Compare this Ext object with another for equality. """ - return (isinstance(other, self.__class__) and - self.type == other.type and - self.data == other.data) + return isinstance(other, self.__class__) \ + and self.type == other.type and self.data == other.data def __ne__(self, other): """ @@ -330,14 +329,11 @@ def _pack_ext(obj, fp, options): elif obj_len == 16: fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) elif obj_len < 2**8: - fp.write(b"\xc7" + - struct.pack("BB", obj_len, obj.type & 0xff) + obj.data) + fp.write(b"\xc7" + struct.pack("BB", obj_len, obj.type & 0xff) + obj.data) elif obj_len < 2**16: - fp.write(b"\xc8" + - struct.pack(">HB", obj_len, obj.type & 0xff) + obj.data) + fp.write(b"\xc8" + struct.pack(">HB", obj_len, obj.type & 0xff) + obj.data) elif obj_len < 2**32: - fp.write(b"\xc9" + - struct.pack(">IB", obj_len, obj.type & 0xff) + obj.data) + fp.write(b"\xc9" + struct.pack(">IB", obj_len, obj.type & 0xff) + obj.data) else: raise UnsupportedTypeException("huge ext data") @@ -356,18 +352,14 @@ def _pack_ext_timestamp(obj, fp, options): if microseconds == 0 and 0 <= seconds <= 2**32 - 1: # 32-bit timestamp - fp.write(b"\xd6\xff" + - struct.pack(">I", seconds)) + fp.write(b"\xd6\xff" + struct.pack(">I", seconds)) elif 0 <= seconds <= 2**34 - 1: # 64-bit timestamp value = ((microseconds * 1000) << 34) | seconds - fp.write(b"\xd7\xff" + - struct.pack(">Q", value)) + fp.write(b"\xd7\xff" + struct.pack(">Q", value)) elif -2**63 <= abs(seconds) <= 2**63 - 1: # 96-bit timestamp - fp.write(b"\xc7\x0c\xff" + - struct.pack(">I", microseconds * 1000) + - struct.pack(">q", seconds)) + fp.write(b"\xc7\x0c\xff" + struct.pack(">Iq", microseconds * 1000, seconds)) else: raise UnsupportedTypeException("huge timestamp") @@ -824,8 +816,7 @@ def _unpack_map(code, fp, options): else: raise Exception("logic error, not map: 0x%02x" % ord(code)) - d = {} if not options.get('use_ordered_dict') \ - else collections.OrderedDict() + d = {} if not options.get('use_ordered_dict') else collections.OrderedDict() for _ in xrange(length): # Unpack key k = _unpack(fp, options) From 871a95391da9cd032a7e665de01bca90c2152acf Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 24 Apr 2020 19:26:06 -0500 Subject: [PATCH 064/109] defer Ext object creation in _unpack_ext() --- umsgpack.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index a4cf00b..7ef9c40 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -746,39 +746,36 @@ def _unpack_ext(code, fp, options): ext_type = struct.unpack("b", _read_except(fp, 1))[0] ext_data = _read_except(fp, length) - # Create extension object - ext = Ext(ext_type, ext_data) - # Unpack with ext handler, if we have one ext_handlers = options.get("ext_handlers") - if ext_handlers and ext.type in ext_handlers: - return ext_handlers[ext.type](ext) + if ext_handlers and ext_type in ext_handlers: + return ext_handlers[ext_type](Ext(ext_type, ext_data)) # Timestamp extension - if ext.type == -1: - return _unpack_ext_timestamp(ext, options) + if ext_type == -1: + return _unpack_ext_timestamp(ext_data, options) - return ext + return Ext(ext_type, ext_data) -def _unpack_ext_timestamp(ext, options): - obj_len = len(ext.data) +def _unpack_ext_timestamp(ext_data, options): + obj_len = len(ext_data) if obj_len == 4: # 32-bit timestamp - seconds = struct.unpack(">I", ext.data)[0] + seconds = struct.unpack(">I", ext_data)[0] microseconds = 0 elif obj_len == 8: # 64-bit timestamp - value = struct.unpack(">Q", ext.data)[0] + value = struct.unpack(">Q", ext_data)[0] seconds = value & 0x3ffffffff microseconds = (value >> 34) // 1000 elif obj_len == 12: # 96-bit timestamp - seconds = struct.unpack(">q", ext.data[4:12])[0] - microseconds = struct.unpack(">I", ext.data[0:4])[0] // 1000 + seconds = struct.unpack(">q", ext_data[4:12])[0] + microseconds = struct.unpack(">I", ext_data[0:4])[0] // 1000 else: raise UnsupportedTimestampException( - "unsupported timestamp with data length %d" % len(ext.data)) + "unsupported timestamp with data length %d" % len(ext_data)) return _epoch + datetime.timedelta(seconds=seconds, microseconds=microseconds) From 8aa0ee8d9d2b7c97a2eba5decfcdec5cc4742fcf Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 24 Apr 2020 00:43:15 -0500 Subject: [PATCH 065/109] add ext_serializable() decorator Co-authored-by: Gabe Appleton --- test_umsgpack.py | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ umsgpack.py | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/test_umsgpack.py b/test_umsgpack.py index 31c01e1..512bf6b 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -383,6 +383,7 @@ "DuplicateKeyException", "KeyNotPrimitiveException", "KeyDuplicateException", + "ext_serializable", "pack", "packb", "unpack", @@ -609,6 +610,68 @@ def test_unpack_ext_override(self): unpacked = umsgpack.unpackb(data, ext_handlers=override_ext_handlers) self.assertEqual(unpacked, obj) + def test_ext_serializable(self): + # Register test class + @umsgpack.ext_serializable(0x20) + class CustomComplex: + def __init__(self, real, imag): + self.real = real + self.imag = imag + + def __eq__(self, other): + return self.real == other.real and self.imag == other.imag + + def packb(self): + return struct.pack(" Date: Fri, 24 Apr 2020 00:44:04 -0500 Subject: [PATCH 066/109] add ext_serializable() decorator usage to readme and msgpack.org.md closes #37. --- README.md | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++ msgpack.org.md | 17 +++++++++++ 2 files changed, 100 insertions(+) diff --git a/README.md b/README.md index 869f76e..e8a6a7f 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,23 @@ b'\x01\x02\x03' >>> ``` +Serializing and deserializing application-defined types with `ext_serializable()`: +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... def packb(self): +... return struct.pack(">ii", self.x, self.y) +... @staticmethod +... def unpackb(data): +... return Point(*struct.unpack(">ii", data)) +... +>>> umsgpack.packb(Point(1, 2)) +b'\xd7P\x00\x00\x00\x01\x00\x00\x00\x02' +>>> umsgpack.unpackb(_) +Point(x=1, y=2) +>>> +``` + Serializing and deserializing application-defined types with Ext handlers: ``` python >>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], @@ -113,6 +130,45 @@ b'\x82\xa7compact\xc3\xa6schema\x00' >>> ``` +## Ext Serializable + +The `ext_serializable()` decorator registers application classes for automatic +packing and unpacking with the specified Ext type. The decorator accepts the +Ext type code as an argument. The application class should implement a +`packb()` method that returns serialized bytes, and an `unpackb()` class method +or static method that accepts serialized bytes and returns an instance of the +application class. + +Example for registering, packing, and unpacking a custom class with Ext type +code 0x10: + +``` python +@umsgpack.ext_serializable(0x10) +class Point(object): + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z + + def __str__(self): + return "Point({}, {}, {})".format(self.x, self.y, self.z) + + def packb(self): + return struct.pack(">iii", self.x, self.y, self.z) + + @staticmethod + def unpackb(data): + return Point(*struct.unpack(">iii", data)) + +# Pack +obj = Point(1,2,3) +data = umsgpack.packb(obj) + +# Unpack +obj = umsgpack.unpackb(data) +print(obj) # -> Point(1, 2, 3) +``` + ## Ext Handlers The packing functions accept an optional `ext_handlers` dictionary that maps @@ -301,6 +357,19 @@ If an error occurs during packing, umsgpack will raise an exception derived from >>> ``` +* `NotImplementedError`: Ext serializable class is missing implementation of `packb()`. + + ``` python + >>> @umsgpack.ext_serializable(0x50) + ... class Point(collections.namedtuple('Point', ['x', 'y'])): + ... pass + ... + >>> umsgpack.packb(Point(1, 2)) + ... + NotImplementedError: Ext serializable class is missing implementation of packb() + >>> + ``` + ### Unpacking Exceptions If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a `TypeError` exception. If an error occurs during unpacking, umsgpack will raise an exception derived from `umsgpack.UnpackException`. All possible unpacking exceptions are described below. @@ -399,6 +468,19 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a >>> ``` +* `NotImplementedError`: Ext serializable class is missing implementation of `unpackb()`. + + ``` python + >>> @umsgpack.ext_serializable(0x50) + ... class Point(collections.namedtuple('Point', ['x', 'y'])): + ... pass + ... + >>> umsgpack.unpackb(b'\xd7\x50\x00\x00\x00\x01\x00\x00\x00\x02') + ... + NotImplementedError: Ext serializable class is missing implementation of unpackb() + >>> + ``` + ## Behavior Notes * Python 2 @@ -414,6 +496,7 @@ If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a * The Python `datetime.datetime` type is packed into, and unpacked from, the msgpack `timestamp` format * Note that this Python type only supports microsecond resolution, while the msgpack `timestamp` format supports nanosecond resolution. Timestamps with finer than microsecond resolution will lose precision during unpacking. Users may override the packing and unpacking of the msgpack `timestamp` format with a custom type for alternate behavior. * Both naive and aware timestamp are supported. Naive timestamps are packed as if they are in the UTC timezone. Timestamps are always unpacked as aware `datetime.datetime` objects in the UTC timezone. +* Ext type handlers specified in the optional `ext_handlers` dictionary will override `ext_serializable()` classes during packing and unpacking ## Testing diff --git a/msgpack.org.md b/msgpack.org.md index 009e629..227c7ea 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -80,6 +80,23 @@ b'\x01\x02\x03' >>> ``` +Serializing and deserializing application-defined types with `ext_serializable()`: +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... def packb(self): +... return struct.pack(">ii", self.x, self.y) +... @staticmethod +... def unpackb(data): +... return Point(*struct.unpack(">ii", data)) +... +>>> umsgpack.packb(Point(1, 2)) +b'\xd7P\x00\x00\x00\x01\x00\x00\x00\x02' +>>> umsgpack.unpackb(_) +Point(x=1, y=2) +>>> +``` + Serializing and deserializing application-defined types with Ext handlers: ``` python >>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], From 8599cae6bc970effe069e3ed955af1c64dab9106 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 24 Apr 2020 00:47:07 -0500 Subject: [PATCH 067/109] update copyright years in license for 2020 --- LICENSE | 2 +- umsgpack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 7e330d5..ba6591c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2013-2016 vsergeev / Ivan (Vanya) A. Sergeev + Copyright (c) 2013-2020 vsergeev / Ivan (Vanya) A. Sergeev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/umsgpack.py b/umsgpack.py index 9ea64e7..d099d5e 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -10,7 +10,7 @@ # # MIT License # -# Copyright (c) 2013-2016 vsergeev / Ivan (Vanya) A. Sergeev +# Copyright (c) 2013-2020 vsergeev / Ivan (Vanya) A. Sergeev # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From cb4db3764a8031a76d231ec9f1cefec36a638ef1 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 24 Apr 2020 00:48:31 -0500 Subject: [PATCH 068/109] update version and changelog to v2.6.0 --- CHANGELOG.md | 6 ++++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad392b8..9ec0da9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +* Version 2.6.0 - 04/25/2020 + * Add `use_tuple` option to unpacking functions for unpacking MessagePack arrays into tuples. + * Add `ext_serializable()` decorator for registration of application classes with Ext types for automatic packing and unpacking. + * Contributors + * Gabe Appleton, @gappleto97 - original idea behind 8aa0ee8 in https://github.com/vsergeev/u-msgpack-python/pull/37 + * Version 2.5.2 - 08/15/2019 * Fix DeprecationWarning about using ABCs from 'collections' on Python 3.7. * Contributors diff --git a/setup.py b/setup.py index d4230a2..abcf225 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.5.2', + version='2.6.0', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index d099d5e..c7dddf5 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.5.2 - v at sergeev.io +# u-msgpack-python v2.6.0 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.5.2 - v at sergeev.io +u-msgpack-python v2.6.0 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -54,10 +54,10 @@ else: from collections import Hashable -__version__ = "2.5.2" +__version__ = "2.6.0" "Module version string" -version = (2, 5, 2) +version = (2, 6, 0) "Module version tuple" From 620f6ef14e735a60b38a81cbabecd215c994c912 Mon Sep 17 00:00:00 2001 From: Gabe Appleton Date: Sun, 3 May 2020 19:35:25 -0400 Subject: [PATCH 069/109] add support for packing subclasses of ext_serializable() classes resolves #45. Signed-off-by: Vanya A. Sergeev --- umsgpack.py | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index c7dddf5..f5f152b 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -140,7 +140,8 @@ class InvalidString(bytes): # Ext Serializable Decorator ############################################################################## -_ext_classes = {} +_ext_classes_to_code = {} +_ext_codes_to_class = {} def ext_serializable(ext_type): @@ -159,13 +160,13 @@ def ext_serializable(ext_type): Ext type or class already registered. """ def wrapper(cls): - if ext_type in _ext_classes: - raise ValueError("Ext type 0x{:02x} already registered with class {:s}".format(ext_type, repr(_ext_classes[ext_type]))) - elif cls in _ext_classes: + if ext_type in _ext_codes_to_class: + raise ValueError("Ext type 0x{:02x} already registered with class {:s}".format(ext_type, repr(_ext_codes_to_class[ext_type]))) + elif cls in _ext_classes_to_code: raise ValueError("Class {:s} already registered with Ext type 0x{:02x}".format(repr(cls), ext_type)) - _ext_classes[ext_type] = cls - _ext_classes[cls] = ext_type + _ext_codes_to_class[ext_type] = cls + _ext_classes_to_code[cls] = ext_type return cls @@ -472,11 +473,19 @@ def _pack2(obj, fp, **options): _pack_nil(obj, fp, options) elif ext_handlers and obj.__class__ in ext_handlers: _pack_ext(ext_handlers[obj.__class__](obj), fp, options) - elif obj.__class__ in _ext_classes: + elif obj.__class__ in _ext_classes_to_code: try: - _pack_ext(Ext(_ext_classes[obj.__class__], obj.packb()), fp, options) + _pack_ext(Ext(_ext_classes_to_code[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) + elif isinstance(obj, tuple(_ext_classes_to_code)): + for cls in _ext_classes_to_code: + if isinstance(obj, cls): + try: + _pack_ext(Ext(_ext_classes_to_code[cls], obj.packb()), fp, options) + break + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(cls))) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) elif isinstance(obj, (int, long)): @@ -549,11 +558,19 @@ def _pack3(obj, fp, **options): _pack_nil(obj, fp, options) elif ext_handlers and obj.__class__ in ext_handlers: _pack_ext(ext_handlers[obj.__class__](obj), fp, options) - elif obj.__class__ in _ext_classes: + elif obj.__class__ in _ext_classes_to_code: try: - _pack_ext(Ext(_ext_classes[obj.__class__], obj.packb()), fp, options) + _pack_ext(Ext(_ext_classes_to_code[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) + elif isinstance(obj, tuple(_ext_classes_to_code)): + for cls in _ext_classes_to_code: + if isinstance(obj, cls): + try: + _pack_ext(Ext(_ext_classes_to_code[cls], obj.packb()), fp, options) + break + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(cls))) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) elif isinstance(obj, int): @@ -799,11 +816,11 @@ def _unpack_ext(code, fp, options): return ext_handlers[ext_type](Ext(ext_type, ext_data)) # Unpack with ext classes, if type is registered - if ext_type in _ext_classes: + if ext_type in _ext_codes_to_class: try: - return _ext_classes[ext_type].unpackb(ext_data) + return _ext_codes_to_class[ext_type].unpackb(ext_data) except AttributeError: - raise NotImplementedError("Ext serializable class {:s} is missing implementation of unpackb()".format(repr(_ext_classes[ext_type]))) + raise NotImplementedError("Ext serializable class {:s} is missing implementation of unpackb()".format(repr(_ext_codes_to_class[ext_type]))) # Timestamp extension if ext_type == -1: From fce7f63fbcb2f5449fd7799d980d5c58f01c66f9 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 31 Jul 2020 03:34:16 -0500 Subject: [PATCH 070/109] rename ext_serializable() mapping dictionaries --- umsgpack.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index f5f152b..b4a1cd6 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -140,8 +140,8 @@ class InvalidString(bytes): # Ext Serializable Decorator ############################################################################## -_ext_classes_to_code = {} -_ext_codes_to_class = {} +_ext_class_to_type = {} +_ext_type_to_class = {} def ext_serializable(ext_type): @@ -160,13 +160,13 @@ def ext_serializable(ext_type): Ext type or class already registered. """ def wrapper(cls): - if ext_type in _ext_codes_to_class: - raise ValueError("Ext type 0x{:02x} already registered with class {:s}".format(ext_type, repr(_ext_codes_to_class[ext_type]))) - elif cls in _ext_classes_to_code: + if ext_type in _ext_type_to_class: + raise ValueError("Ext type 0x{:02x} already registered with class {:s}".format(ext_type, repr(_ext_type_to_class[ext_type]))) + elif cls in _ext_class_to_type: raise ValueError("Class {:s} already registered with Ext type 0x{:02x}".format(repr(cls), ext_type)) - _ext_codes_to_class[ext_type] = cls - _ext_classes_to_code[cls] = ext_type + _ext_type_to_class[ext_type] = cls + _ext_class_to_type[cls] = ext_type return cls @@ -473,16 +473,16 @@ def _pack2(obj, fp, **options): _pack_nil(obj, fp, options) elif ext_handlers and obj.__class__ in ext_handlers: _pack_ext(ext_handlers[obj.__class__](obj), fp, options) - elif obj.__class__ in _ext_classes_to_code: + elif obj.__class__ in _ext_class_to_type: try: - _pack_ext(Ext(_ext_classes_to_code[obj.__class__], obj.packb()), fp, options) + _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) - elif isinstance(obj, tuple(_ext_classes_to_code)): - for cls in _ext_classes_to_code: + elif isinstance(obj, tuple(_ext_class_to_type)): + for cls in _ext_class_to_type: if isinstance(obj, cls): try: - _pack_ext(Ext(_ext_classes_to_code[cls], obj.packb()), fp, options) + _pack_ext(Ext(_ext_class_to_type[cls], obj.packb()), fp, options) break except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(cls))) @@ -558,16 +558,16 @@ def _pack3(obj, fp, **options): _pack_nil(obj, fp, options) elif ext_handlers and obj.__class__ in ext_handlers: _pack_ext(ext_handlers[obj.__class__](obj), fp, options) - elif obj.__class__ in _ext_classes_to_code: + elif obj.__class__ in _ext_class_to_type: try: - _pack_ext(Ext(_ext_classes_to_code[obj.__class__], obj.packb()), fp, options) + _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) - elif isinstance(obj, tuple(_ext_classes_to_code)): - for cls in _ext_classes_to_code: + elif isinstance(obj, tuple(_ext_class_to_type)): + for cls in _ext_class_to_type: if isinstance(obj, cls): try: - _pack_ext(Ext(_ext_classes_to_code[cls], obj.packb()), fp, options) + _pack_ext(Ext(_ext_class_to_type[cls], obj.packb()), fp, options) break except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(cls))) @@ -816,11 +816,11 @@ def _unpack_ext(code, fp, options): return ext_handlers[ext_type](Ext(ext_type, ext_data)) # Unpack with ext classes, if type is registered - if ext_type in _ext_codes_to_class: + if ext_type in _ext_type_to_class: try: - return _ext_codes_to_class[ext_type].unpackb(ext_data) + return _ext_type_to_class[ext_type].unpackb(ext_data) except AttributeError: - raise NotImplementedError("Ext serializable class {:s} is missing implementation of unpackb()".format(repr(_ext_codes_to_class[ext_type]))) + raise NotImplementedError("Ext serializable class {:s} is missing implementation of unpackb()".format(repr(_ext_type_to_class[ext_type]))) # Timestamp extension if ext_type == -1: From b36181ef62f05a857d56bdc8552dfa58362b9607 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 31 Jul 2020 03:37:30 -0500 Subject: [PATCH 071/109] refactor ext_serializable() superclass search in packing --- umsgpack.py | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index b4a1cd6..9e63605 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -478,14 +478,16 @@ def _pack2(obj, fp, **options): _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) - elif isinstance(obj, tuple(_ext_class_to_type)): - for cls in _ext_class_to_type: - if isinstance(obj, cls): - try: - _pack_ext(Ext(_ext_class_to_type[cls], obj.packb()), fp, options) - break - except AttributeError: - raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(cls))) + elif _ext_class_to_type: + # Linear search for superclass + t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) + if t: + try: + _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) + else: + raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) elif isinstance(obj, (int, long)): @@ -563,14 +565,16 @@ def _pack3(obj, fp, **options): _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) - elif isinstance(obj, tuple(_ext_class_to_type)): - for cls in _ext_class_to_type: - if isinstance(obj, cls): - try: - _pack_ext(Ext(_ext_class_to_type[cls], obj.packb()), fp, options) - break - except AttributeError: - raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(cls))) + elif _ext_class_to_type: + # Linear search for superclass + t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) + if t: + try: + _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) + else: + raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) elif isinstance(obj, int): From e6e8566cadb4054b7f93ed810b555ff3f72007d2 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 31 Jul 2020 03:38:16 -0500 Subject: [PATCH 072/109] reorder priority of ext_serializable() superclass search in packing after ext_handlers superclass search. --- umsgpack.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index 9e63605..8313e77 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -478,16 +478,6 @@ def _pack2(obj, fp, **options): _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) - elif _ext_class_to_type: - # Linear search for superclass - t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) - if t: - try: - _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) - except AttributeError: - raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) - else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) elif isinstance(obj, (int, long)): @@ -518,6 +508,16 @@ def _pack2(obj, fp, **options): else: raise UnsupportedTypeException( "unsupported type: %s" % str(type(obj))) + elif _ext_class_to_type: + # Linear search for superclass + t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) + if t: + try: + _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) + else: + raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) else: raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) @@ -565,16 +565,6 @@ def _pack3(obj, fp, **options): _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) - elif _ext_class_to_type: - # Linear search for superclass - t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) - if t: - try: - _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) - except AttributeError: - raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) - else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) elif isinstance(obj, int): @@ -605,6 +595,16 @@ def _pack3(obj, fp, **options): else: raise UnsupportedTypeException( "unsupported type: %s" % str(type(obj))) + elif _ext_class_to_type: + # Linear search for superclass + t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) + if t: + try: + _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) + else: + raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) else: raise UnsupportedTypeException( "unsupported type: %s" % str(type(obj))) From a5d7d40be6072b7df00de31c37725a7ace4c9b6f Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 31 Jul 2020 03:39:32 -0500 Subject: [PATCH 073/109] fix deregistration of Ext serializable classes in unit tests --- test_umsgpack.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index 512bf6b..8476f08 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -669,8 +669,10 @@ class IncompleteClass: with self.assertRaises(NotImplementedError): umsgpack.unpackb(b"\xd4\x21\x00") - # Unregister Ext serializable classes for future tests - umsgpack._ext_classes = {} + # Unregister Ext serializable classes to prevent interference with + # subsequent tests + umsgpack._ext_classes_to_code = {} + umsgpack._ext_code_to_classes = {} def test_streaming_writer(self): # Try first composite test vector From c9c5c7557396f921ef82b2585e8a323d1c3421ae Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 1 Aug 2020 00:22:47 -0500 Subject: [PATCH 074/109] add ext handler subclass test to unit tests --- test_umsgpack.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test_umsgpack.py b/test_umsgpack.py index 8476f08..fcb70a7 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -610,6 +610,31 @@ def test_unpack_ext_override(self): unpacked = umsgpack.unpackb(data, ext_handlers=override_ext_handlers) self.assertEqual(unpacked, obj) + def test_ext_handlers_subclass(self): + class Rectangle: + def __init__(self, length, width): + self.length = length + self.width = width + + def __eq__(self, other): + return self.length == other.length and self.width == other.width + + class Square(Rectangle): + def __init__(self, width): + Rectangle.__init__(self, width, width) + + # Test pack (packs base class) + packed = umsgpack.packb(Square(5), ext_handlers={ + Rectangle: lambda obj: umsgpack.Ext(0x10, umsgpack.packb([obj.length, obj.width])), + }) + self.assertEqual(packed, b"\xc7\x03\x10\x92\x05\x05") + + # Test unpack (unpacks base class) + unpacked = umsgpack.unpackb(packed, ext_handlers={ + 0x10: lambda ext: Rectangle(*umsgpack.unpackb(ext.data)), + }) + self.assertEqual(unpacked, Rectangle(5, 5)) + def test_ext_serializable(self): # Register test class @umsgpack.ext_serializable(0x20) From 77d322d77988affe52d43ca76a954d1efdf4cb58 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 1 Aug 2020 00:23:07 -0500 Subject: [PATCH 075/109] add ext_serializable() subclass test to unit tests --- test_umsgpack.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test_umsgpack.py b/test_umsgpack.py index fcb70a7..2f53dde 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -699,6 +699,40 @@ class IncompleteClass: umsgpack._ext_classes_to_code = {} umsgpack._ext_code_to_classes = {} + def test_ext_serializable_subclass(self): + @umsgpack.ext_serializable(0x10) + class Rectangle: + def __init__(self, length, width): + self.length = length + self.width = width + + def __eq__(self, other): + return self.length == other.length and self.width == other.width + + def packb(self): + return umsgpack.packb([self.length, self.width]) + + @classmethod + def unpackb(cls, data): + return cls(*umsgpack.unpackb(data)) + + class Square(Rectangle): + def __init__(self, width): + Rectangle.__init__(self, width, width) + + # Test pack (packs base class) + packed = umsgpack.packb(Square(5)) + self.assertEqual(packed, b"\xc7\x03\x10\x92\x05\x05") + + # Test unpack (unpacks base class) + unpacked = umsgpack.unpackb(packed) + self.assertEqual(unpacked, Rectangle(5, 5)) + + # Unregister Ext serializable classes to prevent interference with + # subsequent tests + umsgpack._ext_classes_to_code = {} + umsgpack._ext_code_to_classes = {} + def test_streaming_writer(self): # Try first composite test vector (_, obj, data) = composite_test_vectors[0] From 80fe3cc2b9472751adc5d6f3ab2c7712adc92e0b Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 1 Aug 2020 00:30:47 -0500 Subject: [PATCH 076/109] update version and changelog to v2.7.0 --- CHANGELOG.md | 5 +++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ec0da9..355e351 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +* Version 2.7.0 - 08/01/2020 + * Add support for packing subclasses of `ext_serializable()` application classes. + * Contributors + * Gabe Appleton, @gappleto97 - 620f6ef + * Version 2.6.0 - 04/25/2020 * Add `use_tuple` option to unpacking functions for unpacking MessagePack arrays into tuples. * Add `ext_serializable()` decorator for registration of application classes with Ext types for automatic packing and unpacking. diff --git a/setup.py b/setup.py index abcf225..7fdafa4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.6.0', + version='2.7.0', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index 8313e77..ff53036 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.6.0 - v at sergeev.io +# u-msgpack-python v2.7.0 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.6.0 - v at sergeev.io +u-msgpack-python v2.7.0 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -54,10 +54,10 @@ else: from collections import Hashable -__version__ = "2.6.0" +__version__ = "2.7.0" "Module version string" -version = (2, 6, 0) +version = (2, 7, 0) "Module version tuple" From 6a1771f3760909d112124d1626877e3dbe49a61f Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:03:46 -0500 Subject: [PATCH 077/109] format ext type value as signed decimal instead of hex --- umsgpack.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index ff53036..1596d35 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -81,12 +81,12 @@ def __init__(self, type, data): data: application-defined data byte array Example: - >>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") + >>> foo = umsgpack.Ext(5, b"\x01\x02\x03") >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) '\x82\xa7awesome\xc3\xadspecial stuff\xc7\x03\x05\x01\x02\x03' >>> bar = umsgpack.unpackb(_) >>> print(bar["special stuff"]) - Ext Object (Type: 0x05, Data: 01 02 03) + Ext Object (Type: 5, Data: 01 02 03) >>> """ # Check type is type int @@ -117,7 +117,7 @@ def __str__(self): """ String representation of this Ext object. """ - s = "Ext Object (Type: 0x%02x, Data: " % self.type + s = "Ext Object (Type: %d, Data: " % self.type s += " ".join(["0x%02x" % ord(self.data[i:i + 1]) for i in xrange(min(len(self.data), 8))]) if len(self.data) > 8: @@ -161,9 +161,9 @@ def ext_serializable(ext_type): """ def wrapper(cls): if ext_type in _ext_type_to_class: - raise ValueError("Ext type 0x{:02x} already registered with class {:s}".format(ext_type, repr(_ext_type_to_class[ext_type]))) + raise ValueError("Ext type {:d} already registered with class {:s}".format(ext_type, repr(_ext_type_to_class[ext_type]))) elif cls in _ext_class_to_type: - raise ValueError("Class {:s} already registered with Ext type 0x{:02x}".format(repr(cls), ext_type)) + raise ValueError("Class {:s} already registered with Ext type {:d}".format(repr(cls), ext_type)) _ext_type_to_class[ext_type] = cls _ext_class_to_type[cls] = ext_type From 9d511bcb355d8cfc59957c09ac705924c6ffb507 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:06:54 -0500 Subject: [PATCH 078/109] update ext type value formatting in readme and msgpack.org.md --- README.md | 6 +++--- msgpack.org.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e8a6a7f..bb04cbb 100644 --- a/README.md +++ b/README.md @@ -64,14 +64,14 @@ Streaming serialization with file-like objects: Serializing and deserializing a raw Ext type: ``` python ->>> # Create an Ext object with type 0x05 and data b"\x01\x02\x03" -... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") +>>> # Create an Ext object with type 5 and data b"\x01\x02\x03" +... foo = umsgpack.Ext(5, b"\x01\x02\x03") >>> umsgpack.packb({u"stuff": foo, u"awesome": True}) b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' >>> >>> bar = umsgpack.unpackb(_) >>> print(bar['stuff']) -Ext Object (Type: 0x05, Data: 0x01 0x02 0x03) +Ext Object (Type: 5, Data: 0x01 0x02 0x03) >>> bar['stuff'].type 5 >>> bar['stuff'].data diff --git a/msgpack.org.md b/msgpack.org.md index 227c7ea..3117777 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -65,14 +65,14 @@ Streaming serialization with file-like objects: Serializing and deserializing a raw Ext type: ``` python ->>> # Create an Ext object with type 0x05 and data b"\x01\x02\x03" -... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") +>>> # Create an Ext object with type 5 and data b"\x01\x02\x03" +... foo = umsgpack.Ext(5, b"\x01\x02\x03") >>> umsgpack.packb({u"stuff": foo, u"awesome": True}) b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' >>> >>> bar = umsgpack.unpackb(_) >>> print(bar['stuff']) -Ext Object (Type: 0x05, Data: 0x01 0x02 0x03) +Ext Object (Type: 5, Data: 0x01 0x02 0x03) >>> bar['stuff'].type 5 >>> bar['stuff'].data From c3a6a9529d396280739a60cd891f29edba201753 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:07:08 -0500 Subject: [PATCH 079/109] add ext type value validation to Ext class and ext_serializable() decorator resolves #46. --- test_umsgpack.py | 18 ++++++++++++++++++ umsgpack.py | 24 +++++++++++++++++++++--- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index 2f53dde..f6bedd2 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -539,12 +539,20 @@ def test_unpack_tuple(self): self.assertEqual(umsgpack.unpackb(data, use_tuple=True), obj_tuple) def test_ext_exceptions(self): + # Test invalid Ext type type with self.assertRaises(TypeError): _ = umsgpack.Ext(5.0, b"") + # Test invalid data type with self.assertRaises(TypeError): _ = umsgpack.Ext(0, u"unicode string") + # Test out of range Ext type value + with self.assertRaises(ValueError): + _ = umsgpack.Ext(-129, b"data") + with self.assertRaises(ValueError): + _ = umsgpack.Ext(128, b"data") + def test_pack_ext_handler(self): for (name, obj, data) in ext_handlers_test_vectors: obj_repr = repr(obj) @@ -681,6 +689,16 @@ def unpackb(cls, data): class DummyClass: pass + # Test out of range Ext type value + with self.assertRaises(ValueError): + @umsgpack.ext_serializable(-129) + class DummyClass2: + pass + with self.assertRaises(ValueError): + @umsgpack.ext_serializable(128) + class DummyClass3: + pass + # Register class with missing packb() and unpackb() @umsgpack.ext_serializable(0x21) class IncompleteClass: diff --git a/umsgpack.py b/umsgpack.py index 1596d35..e6487ab 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -80,6 +80,13 @@ def __init__(self, type, data): type: application-defined type integer data: application-defined data byte array + TypeError: + Type is not an integer. + ValueError: + Type is out of range of -128 to 127. + TypeError:: + Data is not type 'bytes' (Python 3) or not type 'str' (Python 2). + Example: >>> foo = umsgpack.Ext(5, b"\x01\x02\x03") >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) @@ -89,14 +96,17 @@ def __init__(self, type, data): Ext Object (Type: 5, Data: 01 02 03) >>> """ - # Check type is type int + # Check type is type int and in range if not isinstance(type, int): raise TypeError("ext type is not type integer") - # Check data is type bytes + elif not (-2**7 <= type <= 2**7 - 1): + raise ValueError("ext type value {:d} is out of range (-128 to 127)".format(type)) + # Check data is type bytes or str elif sys.version_info[0] == 3 and not isinstance(data, bytes): raise TypeError("ext data is not type \'bytes\'") elif sys.version_info[0] == 2 and not isinstance(data, str): raise TypeError("ext data is not type \'str\'") + self.type = type self.data = data @@ -156,11 +166,19 @@ def ext_serializable(ext_type): ext_type: application-defined Ext type code Raises: + TypeError: + Ext type is not an integer. + ValueError: + Ext type is out of range of -128 to 127. ValueError: Ext type or class already registered. """ def wrapper(cls): - if ext_type in _ext_type_to_class: + if not isinstance(ext_type, int): + raise TypeError("Ext type is not type integer") + elif not (-2**7 <= ext_type <= 2**7 - 1): + raise ValueError("Ext type value {:d} is out of range of -128 to 127".format(ext_type)) + elif ext_type in _ext_type_to_class: raise ValueError("Ext type {:d} already registered with class {:s}".format(ext_type, repr(_ext_type_to_class[ext_type]))) elif cls in _ext_class_to_type: raise ValueError("Class {:s} already registered with Ext type {:d}".format(repr(cls), ext_type)) From e1a16d4c3c1efce7fdeeba2216e02cd29c7c3f66 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:26:49 -0500 Subject: [PATCH 080/109] change to .format() strings in Ext string representation --- umsgpack.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index e6487ab..fde2342 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -127,8 +127,8 @@ def __str__(self): """ String representation of this Ext object. """ - s = "Ext Object (Type: %d, Data: " % self.type - s += " ".join(["0x%02x" % ord(self.data[i:i + 1]) + s = "Ext Object (Type: {:d}, Data: ".format(self.type) + s += " ".join(["0x{:02}".format(ord(self.data[i:i + 1])) for i in xrange(min(len(self.data), 8))]) if len(self.data) > 8: s += " ..." From 570c082bf92376da051903326a0f5a5278b70de2 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:27:04 -0500 Subject: [PATCH 081/109] change to .format() strings in exceptions --- umsgpack.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index fde2342..4d63a1e 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -525,7 +525,7 @@ def _pack2(obj, fp, **options): _pack_ext(ext_handlers[t](obj), fp, options) else: raise UnsupportedTypeException( - "unsupported type: %s" % str(type(obj))) + "unsupported type: {:s}".format(str(type(obj)))) elif _ext_class_to_type: # Linear search for superclass t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) @@ -535,9 +535,9 @@ def _pack2(obj, fp, **options): except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj)))) else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj)))) # Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type @@ -612,7 +612,7 @@ def _pack3(obj, fp, **options): _pack_ext(ext_handlers[t](obj), fp, options) else: raise UnsupportedTypeException( - "unsupported type: %s" % str(type(obj))) + "unsupported type: {:s}".format(str(type(obj)))) elif _ext_class_to_type: # Linear search for superclass t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) @@ -622,10 +622,10 @@ def _pack3(obj, fp, **options): except AttributeError: raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) + raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj)))) else: raise UnsupportedTypeException( - "unsupported type: %s" % str(type(obj))) + "unsupported type: {:s}".format(str(type(obj)))) def _packb2(obj, **options): @@ -737,21 +737,21 @@ def _unpack_integer(code, fp, options): return struct.unpack(">I", _read_except(fp, 4))[0] elif code == b'\xcf': return struct.unpack(">Q", _read_except(fp, 8))[0] - raise Exception("logic error, not int: 0x%02x" % ord(code)) + raise Exception("logic error, not int: 0x{:02x}".format(ord(code))) def _unpack_reserved(code, fp, options): if code == b'\xc1': raise ReservedCodeException( - "encountered reserved code: 0x%02x" % ord(code)) + "encountered reserved code: 0x{:02x}".format(ord(code))) raise Exception( - "logic error, not reserved code: 0x%02x" % ord(code)) + "logic error, not reserved code: 0x{:02x}".format(ord(code))) def _unpack_nil(code, fp, options): if code == b'\xc0': return None - raise Exception("logic error, not nil: 0x%02x" % ord(code)) + raise Exception("logic error, not nil: 0x{:02x}".format(ord(code))) def _unpack_boolean(code, fp, options): @@ -759,7 +759,7 @@ def _unpack_boolean(code, fp, options): return False elif code == b'\xc3': return True - raise Exception("logic error, not boolean: 0x%02x" % ord(code)) + raise Exception("logic error, not boolean: 0x{:02x}".format(ord(code))) def _unpack_float(code, fp, options): @@ -767,7 +767,7 @@ def _unpack_float(code, fp, options): return struct.unpack(">f", _read_except(fp, 4))[0] elif code == b'\xcb': return struct.unpack(">d", _read_except(fp, 8))[0] - raise Exception("logic error, not float: 0x%02x" % ord(code)) + raise Exception("logic error, not float: 0x{:02x}".format(ord(code))) def _unpack_string(code, fp, options): @@ -780,7 +780,7 @@ def _unpack_string(code, fp, options): elif code == b'\xdb': length = struct.unpack(">I", _read_except(fp, 4))[0] else: - raise Exception("logic error, not string: 0x%02x" % ord(code)) + raise Exception("logic error, not string: 0x{:02x}".format(ord(code))) # Always return raw bytes in compatibility mode global compatibility @@ -804,7 +804,7 @@ def _unpack_binary(code, fp, options): elif code == b'\xc6': length = struct.unpack(">I", _read_except(fp, 4))[0] else: - raise Exception("logic error, not binary: 0x%02x" % ord(code)) + raise Exception("logic error, not binary: 0x{:02x}".format(ord(code))) return _read_except(fp, length) @@ -827,7 +827,7 @@ def _unpack_ext(code, fp, options): elif code == b'\xc9': length = struct.unpack(">I", _read_except(fp, 4))[0] else: - raise Exception("logic error, not ext: 0x%02x" % ord(code)) + raise Exception("logic error, not ext: 0x{:02x}".format(ord(code))) ext_type = struct.unpack("b", _read_except(fp, 1))[0] ext_data = _read_except(fp, length) @@ -868,7 +868,7 @@ def _unpack_ext_timestamp(ext_data, options): microseconds = struct.unpack(">I", ext_data[0:4])[0] // 1000 else: raise UnsupportedTimestampException( - "unsupported timestamp with data length %d" % len(ext_data)) + "unsupported timestamp with data length {:d}".format(len(ext_data))) return _epoch + datetime.timedelta(seconds=seconds, microseconds=microseconds) @@ -882,7 +882,7 @@ def _unpack_array(code, fp, options): elif code == b'\xdd': length = struct.unpack(">I", _read_except(fp, 4))[0] else: - raise Exception("logic error, not array: 0x%02x" % ord(code)) + raise Exception("logic error, not array: 0x{:02x}".format(ord(code))) if options.get('use_tuple'): return tuple((_unpack(fp, options) for i in xrange(length))) @@ -904,7 +904,7 @@ def _unpack_map(code, fp, options): elif code == b'\xdf': length = struct.unpack(">I", _read_except(fp, 4))[0] else: - raise Exception("logic error, not map: 0x%02x" % ord(code)) + raise Exception("logic error, not map: 0x{:02x}".format(ord(code))) d = {} if not options.get('use_ordered_dict') else collections.OrderedDict() for _ in xrange(length): @@ -916,10 +916,10 @@ def _unpack_map(code, fp, options): k = _deep_list_to_tuple(k) elif not isinstance(k, Hashable): raise UnhashableKeyException( - "encountered unhashable key: %s, %s" % (str(k), str(type(k)))) + "encountered unhashable key: \"{:s}\" ({:s})".format(str(k), str(type(k)))) elif k in d: raise DuplicateKeyException( - "encountered duplicate key: %s, %s" % (str(k), str(type(k)))) + "encountered duplicate key: \"{:s}\" ({:s})".format(str(k), str(type(k)))) # Unpack value v = _unpack(fp, options) @@ -928,7 +928,7 @@ def _unpack_map(code, fp, options): d[k] = v except TypeError: raise UnhashableKeyException( - "encountered unhashable key: %s" % str(k)) + "encountered unhashable key: \"{:s}\"".format(str(k))) return d From 36aede91aa9ead95bc84d0838e649da3ac866f84 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:33:22 -0500 Subject: [PATCH 082/109] change to .format() strings in unit test reporting --- test_umsgpack.py | 58 ++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index f6bedd2..ba9d771 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -404,24 +404,24 @@ class TestUmsgpack(unittest.TestCase): def test_pack_single(self): for (name, obj, data) in single_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) self.assertEqual(umsgpack.packb(obj), data) def test_pack_composite(self): for (name, obj, data) in composite_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) self.assertEqual(umsgpack.packb(obj), data) def test_pack_exceptions(self): for (name, obj, exception) in pack_exception_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) with self.assertRaises(exception): umsgpack.packb(obj) @@ -429,8 +429,8 @@ def test_pack_exceptions(self): def test_unpack_single(self): for (name, obj, data) in single_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) unpacked = umsgpack.unpackb(data) @@ -452,14 +452,14 @@ def test_unpack_single(self): def test_unpack_composite(self): for (name, obj, data) in composite_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) self.assertEqual(umsgpack.unpackb(data), obj) def test_unpack_exceptions(self): for (name, data, exception) in unpack_exception_test_vectors: - print("\tTesting %s" % name) + print("\tTesting {:s}".format(name)) with self.assertRaises(exception): umsgpack.unpackb(data) @@ -469,8 +469,8 @@ def test_pack_compatibility(self): for (name, obj, data) in compatibility_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) self.assertEqual(umsgpack.packb(obj), data) @@ -481,8 +481,8 @@ def test_unpack_compatibility(self): for (name, obj, data) in compatibility_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) unpacked = umsgpack.unpackb(data) @@ -556,8 +556,8 @@ def test_ext_exceptions(self): def test_pack_ext_handler(self): for (name, obj, data) in ext_handlers_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) packed = umsgpack.packb(obj, ext_handlers=ext_handlers) self.assertEqual(packed, data) @@ -565,8 +565,8 @@ def test_pack_ext_handler(self): def test_unpack_ext_handler(self): for (name, obj, data) in ext_handlers_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) unpacked = umsgpack.unpackb(data, ext_handlers=ext_handlers) self.assertEqual(unpacked, obj) @@ -574,8 +574,8 @@ def test_unpack_ext_handler(self): def test_pack_force_float_precision(self): for ((name, obj, data), precision) in zip(float_precision_test_vectors, ["single", "double"]): obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) packed = umsgpack.packb(obj, force_float_precision=precision) self.assertEqual(packed, data) @@ -583,8 +583,8 @@ def test_pack_force_float_precision(self): def test_pack_naive_timestamp(self): for (name, obj, data, _) in naive_timestamp_test_vectors: obj_repr = repr(obj) - print("\t Testing %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) packed = umsgpack.packb(obj) self.assertEqual(packed, data) @@ -592,8 +592,8 @@ def test_pack_naive_timestamp(self): def test_unpack_naive_timestamp(self): for (name, _, data, obj) in naive_timestamp_test_vectors: obj_repr = repr(obj) - print("\t Testing %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) unpacked = umsgpack.unpackb(data) self.assertEqual(unpacked, obj) @@ -602,8 +602,8 @@ def test_pack_ext_override(self): # Test overridden packing of datetime.datetime (name, obj, data) = override_ext_handlers_test_vectors[0] obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) packed = umsgpack.packb(obj, ext_handlers=override_ext_handlers) self.assertEqual(packed, data) @@ -612,8 +612,8 @@ def test_unpack_ext_override(self): # Test overridden unpacking of Ext type -1 (name, obj, data) = override_ext_handlers_test_vectors[1] obj_repr = repr(obj) - print("\tTesting %s: object %s" % - (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) unpacked = umsgpack.unpackb(data, ext_handlers=override_ext_handlers) self.assertEqual(unpacked, obj) From a827029734f688899ab1515cc9d0762a319611c6 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:34:46 -0500 Subject: [PATCH 083/109] simplify filter expressions in unit tests --- test_umsgpack.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test_umsgpack.py b/test_umsgpack.py index ba9d771..1652d60 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -766,11 +766,10 @@ def test_streaming_reader(self): def test_namespacing(self): # Get a list of global variables from umsgpack module - exported_vars = list(filter(lambda x: not x.startswith("_"), - dir(umsgpack))) + exported_vars = list([x for x in dir(umsgpack) if not x.startswith("_")]) # Ignore imports - exported_vars = list(filter(lambda x: x != "struct" and x != "collections" and x != "datetime" and x != - "sys" and x != "io" and x != "xrange" and x != "Hashable", exported_vars)) + exported_vars = list([x for x in exported_vars if x != "struct" and x != "collections" and x != "datetime" and x != + "sys" and x != "io" and x != "xrange" and x != "Hashable"]) self.assertTrue(len(exported_vars) == len(exported_vars_test_vector)) for var in exported_vars_test_vector: From e0c4000224d3d8e232fb9e94b5a07ed2c17ca802 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 24 Oct 2020 22:42:26 -0500 Subject: [PATCH 084/109] update version and changelog to v2.7.1 --- CHANGELOG.md | 4 ++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 355e351..39d2bd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +* Version 2.7.1 - 10/24/2020 + * Add Ext type value validation to Ext class and `ext_serializable()` decorator. + * Change string formatting from `%` to `.format()` throughout codebase. + * Version 2.7.0 - 08/01/2020 * Add support for packing subclasses of `ext_serializable()` application classes. * Contributors diff --git a/setup.py b/setup.py index 7fdafa4..7f040eb 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.7.0', + version='2.7.1', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index 4d63a1e..c6fa1e7 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.7.0 - v at sergeev.io +# u-msgpack-python v2.7.1 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.7.0 - v at sergeev.io +u-msgpack-python v2.7.1 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -54,10 +54,10 @@ else: from collections import Hashable -__version__ = "2.7.0" +__version__ = "2.7.1" "Module version string" -version = (2, 7, 0) +version = (2, 7, 1) "Module version tuple" From d2e5f36a18460ecad767623163dac78f8bbad433 Mon Sep 17 00:00:00 2001 From: Matt Smith Date: Fri, 24 Sep 2021 09:44:49 +0100 Subject: [PATCH 085/109] replace dash-separated options in setup.cfg Recent versions of setuptools report that options with names separated by a dash (e.g. 'home-page') are deprecated and support will be removed in later versions. Signed-off-by: Vanya A. Sergeev --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 768d2bc..689fa54 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -description-file = README.md +description_file = README.md [bdist_wheel] universal = True From 6d38e3f50428641689c0d8c87dc97346d058af0a Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 2 Apr 2022 15:34:52 -0500 Subject: [PATCH 086/109] update dist to focal in travis config --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0b6e534..26f74e6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ sudo: false -dist: xenial +dist: focal language: python install: pip install tox script: tox From 14a7e976287adc4b8ed8ed06cb2d96fd64229bfa Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sat, 2 Apr 2022 15:42:07 -0500 Subject: [PATCH 087/109] update python versions in travis config --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 26f74e6..ba6728f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,11 @@ matrix: env: TOXENV=py37 - python: 3.8 env: TOXENV=py38 - - python: pypy2.7-6.0 + - python: 3.9 + env: TOXENV=py39 + - python: 3.10 + env: TOXENV=py310 + - python: pypy2.7-7.3.1 env: TOXENV=pypy - - python: pypy3.5-6.0 + - python: pypy3.6-7.3.1 env: TOXENV=pypy3 From 309b1a91f4c60c52ecb041db0ccfc525ea27f634 Mon Sep 17 00:00:00 2001 From: Stefan Ring Date: Wed, 19 Oct 2022 11:52:35 +0200 Subject: [PATCH 088/109] fix hex formatting of data bytes in Ext string representation Signed-off-by: Vanya A. Sergeev --- umsgpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/umsgpack.py b/umsgpack.py index c6fa1e7..43ea874 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -128,7 +128,7 @@ def __str__(self): String representation of this Ext object. """ s = "Ext Object (Type: {:d}, Data: ".format(self.type) - s += " ".join(["0x{:02}".format(ord(self.data[i:i + 1])) + s += " ".join(["0x{:02x}".format(ord(self.data[i:i + 1])) for i in xrange(min(len(self.data), 8))]) if len(self.data) > 8: s += " ..." From 8794887e2a71258ddacb7c435ca6f43304c2a21b Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 8 Nov 2022 00:01:11 -0600 Subject: [PATCH 089/109] update copyright years in license --- LICENSE | 2 +- umsgpack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index ba6591c..8bf1bb8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2013-2020 vsergeev / Ivan (Vanya) A. Sergeev + Copyright (c) 2013-2022 vsergeev / Ivan (Vanya) A. Sergeev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/umsgpack.py b/umsgpack.py index 43ea874..592a573 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -10,7 +10,7 @@ # # MIT License # -# Copyright (c) 2013-2020 vsergeev / Ivan (Vanya) A. Sergeev +# Copyright (c) 2013-2022 vsergeev / Ivan (Vanya) A. Sergeev # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From e5d4eb62e299cac1409e80bfaf5cc566d432c91e Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 8 Nov 2022 00:03:38 -0600 Subject: [PATCH 090/109] update travis-ci link in build status badge in readme and msgpack.org.md --- README.md | 2 +- msgpack.org.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb04cbb..24ebb63 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# u-msgpack-python [![Build Status](https://travis-ci.org/vsergeev/u-msgpack-python.svg?branch=master)](https://travis-ci.org/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) +# u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. diff --git a/msgpack.org.md b/msgpack.org.md index 3117777..ae4c100 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -1,4 +1,4 @@ -# u-msgpack-python [![Build Status](https://travis-ci.org/vsergeev/u-msgpack-python.svg?branch=master)](https://travis-ci.org/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) +# u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). From e8eed4031a6b0ad77c5896d8c884a1d7272f4912 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 8 Nov 2022 00:07:45 -0600 Subject: [PATCH 091/109] update version and changelog to v2.7.2 --- CHANGELOG.md | 5 +++++ setup.py | 2 +- umsgpack.py | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39d2bd8..ad1d433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +* Version 2.7.2 - 11/07/2022 + * Fix hex formatting of data bytes in Ext string representation. + * Contributors + * Stefan Ring, @Ringdingcoder - 309b1a9 + * Version 2.7.1 - 10/24/2020 * Add Ext type value validation to Ext class and `ext_serializable()` decorator. * Change string formatting from `%` to `.format()` throughout codebase. diff --git a/setup.py b/setup.py index 7f040eb..ced0b12 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.7.1', + version='2.7.2', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack.py b/umsgpack.py index 592a573..3d80c55 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.7.1 - v at sergeev.io +# u-msgpack-python v2.7.2 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.7.1 - v at sergeev.io +u-msgpack-python v2.7.2 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -54,10 +54,10 @@ else: from collections import Hashable -__version__ = "2.7.1" +__version__ = "2.7.2" "Module version string" -version = (2, 7, 1) +version = (2, 7, 2) "Module version tuple" From d205f24755225334430d3fc72a5fc4697fe7d9d3 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 9 Apr 2023 17:12:19 -0500 Subject: [PATCH 092/109] add py39, py310, py311 environments to tox config --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 73d8b30..5499181 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, py37, py38, pypy, pypy3 +envlist = py27, py35, py36, py37, py38, py39, py310, py311, pypy, pypy3 skip_missing_interpreters=true [testenv] deps = pytest From 12ed425b4fb0622f18c72cae385162e7fc7bc251 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Sun, 9 Apr 2023 17:12:42 -0500 Subject: [PATCH 093/109] enable py311 tox environment in travis config --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index ba6728f..e295bf7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,8 @@ matrix: env: TOXENV=py39 - python: 3.10 env: TOXENV=py310 + - python: 3.11 + env: TOXENV=py311 - python: pypy2.7-7.3.1 env: TOXENV=pypy - python: pypy3.6-7.3.1 From 931960b89372d1a94aa9734b549cbe87361222de Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 3 May 2023 01:09:56 -0500 Subject: [PATCH 094/109] update copyright years in license --- LICENSE | 2 +- umsgpack.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 8bf1bb8..79056ac 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2013-2022 vsergeev / Ivan (Vanya) A. Sergeev + Copyright (c) 2013-2023 vsergeev / Ivan (Vanya) A. Sergeev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/umsgpack.py b/umsgpack.py index 3d80c55..dcf1918 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -10,7 +10,7 @@ # # MIT License # -# Copyright (c) 2013-2022 vsergeev / Ivan (Vanya) A. Sergeev +# Copyright (c) 2013-2023 vsergeev / Ivan (Vanya) A. Sergeev # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal From 07eb1540c76efcfffa82605e0755c180e2b2aab7 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 3 May 2023 01:10:14 -0500 Subject: [PATCH 095/109] add flake8 configuration --- .flake8 | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..a6578a3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +extend-ignore = E501 From 479cfcc3b54bb3d7f821c1250f937affa10e775e Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 3 May 2023 01:11:07 -0500 Subject: [PATCH 096/109] add flake8 noqa ignores for python2 types --- umsgpack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index dcf1918..af596e9 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -498,15 +498,15 @@ def _pack2(obj, fp, **options): raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) elif isinstance(obj, bool): _pack_boolean(obj, fp, options) - elif isinstance(obj, (int, long)): + elif isinstance(obj, (int, long)): # noqa: F821 _pack_integer(obj, fp, options) elif isinstance(obj, float): _pack_float(obj, fp, options) - elif compatibility and isinstance(obj, unicode): + elif compatibility and isinstance(obj, unicode): # noqa: F821 _pack_oldspec_raw(bytes(obj), fp, options) elif compatibility and isinstance(obj, bytes): _pack_oldspec_raw(obj, fp, options) - elif isinstance(obj, unicode): + elif isinstance(obj, unicode): # noqa: F821 _pack_string(obj, fp, options) elif isinstance(obj, str): _pack_binary(obj, fp, options) From aa532297e7fdb56b52a99ef56beacddacb8c115f Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 10 May 2023 02:18:52 -0500 Subject: [PATCH 097/109] fix UnsupportedTypeException name in pack docstrings --- umsgpack.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index af596e9..e756c27 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -475,7 +475,7 @@ def _pack2(obj, fp, **options): None. Raises: - UnsupportedType(PackException): + UnsupportedTypeException(PackException): Object type not supported for packing. Example: @@ -562,7 +562,7 @@ def _pack3(obj, fp, **options): None. Raises: - UnsupportedType(PackException): + UnsupportedTypeException(PackException): Object type not supported for packing. Example: @@ -648,7 +648,7 @@ def _packb2(obj, **options): A 'str' containing serialized MessagePack bytes. Raises: - UnsupportedType(PackException): + UnsupportedTypeException(PackException): Object type not supported for packing. Example: @@ -681,7 +681,7 @@ def _packb3(obj, **options): A 'bytes' containing serialized MessagePack bytes. Raises: - UnsupportedType(PackException): + UnsupportedTypeException(PackException): Object type not supported for packing. Example: From a4c5ad6de64d2d14a8e7715435fbc5d711082426 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 10 May 2023 02:33:21 -0500 Subject: [PATCH 098/109] improve types and formatting in docstrings --- umsgpack.py | 164 ++++++++++++++++++++++++---------------------------- 1 file changed, 77 insertions(+), 87 deletions(-) diff --git a/umsgpack.py b/umsgpack.py index e756c27..03f39a1 100644 --- a/umsgpack.py +++ b/umsgpack.py @@ -77,24 +77,24 @@ def __init__(self, type, data): Construct a new Ext object. Args: - type: application-defined type integer - data: application-defined data byte array + type (int): application-defined type integer + data (bytes): application-defined data byte array - TypeError: - Type is not an integer. - ValueError: - Type is out of range of -128 to 127. - TypeError:: - Data is not type 'bytes' (Python 3) or not type 'str' (Python 2). + Raises: + TypeError: + Type is not an integer. + ValueError: + Type is out of range of -128 to 127. + TypeError: + Data is not type 'bytes' (Python 3) or not type 'str' (Python 2). Example: - >>> foo = umsgpack.Ext(5, b"\x01\x02\x03") - >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) - '\x82\xa7awesome\xc3\xadspecial stuff\xc7\x03\x05\x01\x02\x03' - >>> bar = umsgpack.unpackb(_) - >>> print(bar["special stuff"]) - Ext Object (Type: 5, Data: 01 02 03) - >>> + >>> foo = umsgpack.Ext(5, b"\\x01\\x02\\x03") + >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) + '\\x82\\xa7awesome\\xc3\\xadspecial stuff\\xc7\\x03\\x05\\x01\\x02\\x03' + >>> bar = umsgpack.unpackb(_) + >>> print(bar["special stuff"]) + Ext Object (Type: 5, Data: 01 02 03) """ # Check type is type int and in range if not isinstance(type, int): @@ -163,7 +163,7 @@ def ext_serializable(ext_type): instance of the application class. Args: - ext_type: application-defined Ext type code + ext_type (int): application-defined Ext type code Raises: TypeError: @@ -266,13 +266,11 @@ class DuplicateKeyException(UnpackException): old MessagePack specification. Example: ->>> umsgpack.compatibility = True ->>> ->>> umsgpack.packb([u"some string", b"some bytes"]) -b'\x92\xabsome string\xaasome bytes' ->>> umsgpack.unpackb(_) -[b'some string', b'some bytes'] ->>> + >>> umsgpack.compatibility = True + >>> umsgpack.packb([u"some string", b"some bytes"]) + b'\\x92\\xabsome string\\xaasome bytes' + >>> umsgpack.unpackb(_) + [b'some string', b'some bytes'] """ ############################################################################## @@ -462,26 +460,25 @@ def _pack2(obj, fp, **options): obj: a Python object fp: a .write()-supporting file-like object - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object force_float_precision (str): "single" to force packing floats as IEEE-754 single-precision floats, "double" to force packing floats as - IEEE-754 double-precision floats. + IEEE-754 double-precision floats Returns: - None. + None Raises: UnsupportedTypeException(PackException): Object type not supported for packing. Example: - >>> f = open('test.bin', 'wb') - >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) - >>> + >>> f = open('test.bin', 'wb') + >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) """ global compatibility @@ -549,26 +546,25 @@ def _pack3(obj, fp, **options): obj: a Python object fp: a .write()-supporting file-like object - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object force_float_precision (str): "single" to force packing floats as IEEE-754 single-precision floats, "double" to force packing floats as - IEEE-754 double-precision floats. + IEEE-754 double-precision floats Returns: - None. + None Raises: UnsupportedTypeException(PackException): Object type not supported for packing. Example: - >>> f = open('test.bin', 'wb') - >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) - >>> + >>> f = open('test.bin', 'wb') + >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) """ global compatibility @@ -635,26 +631,25 @@ def _packb2(obj, **options): Args: obj: a Python object - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object force_float_precision (str): "single" to force packing floats as IEEE-754 single-precision floats, "double" to force packing floats as - IEEE-754 double-precision floats. + IEEE-754 double-precision floats Returns: - A 'str' containing serialized MessagePack bytes. + str: Serialized MessagePack bytes Raises: UnsupportedTypeException(PackException): Object type not supported for packing. Example: - >>> umsgpack.packb({u"compact": True, u"schema": 0}) - '\x82\xa7compact\xc3\xa6schema\x00' - >>> + >>> umsgpack.packb({u"compact": True, u"schema": 0}) + '\\x82\\xa7compact\\xc3\\xa6schema\\x00' """ fp = io.BytesIO() _pack2(obj, fp, **options) @@ -668,26 +663,25 @@ def _packb3(obj, **options): Args: obj: a Python object - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping a custom type to a callable that packs an instance of the type into an Ext object force_float_precision (str): "single" to force packing floats as IEEE-754 single-precision floats, "double" to force packing floats as - IEEE-754 double-precision floats. + IEEE-754 double-precision floats Returns: - A 'bytes' containing serialized MessagePack bytes. + bytes: Serialized MessagePack bytes Raises: UnsupportedTypeException(PackException): Object type not supported for packing. Example: - >>> umsgpack.packb({u"compact": True, u"schema": 0}) - b'\x82\xa7compact\xc3\xa6schema\x00' - >>> + >>> umsgpack.packb({u"compact": True, u"schema": 0}) + b'\\x82\\xa7compact\\xc3\\xa6schema\\x00' """ fp = io.BytesIO() _pack3(obj, fp, **options) @@ -946,20 +940,20 @@ def _unpack2(fp, **options): Args: fp: a .read()-supporting file-like object - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext type to a callable that unpacks an instance of Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) use_tuple (bool): unpacks arrays into tuples, instead of lists (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) + :class:`InvalidString`, for access to the + bytes (default False) Returns: - A Python object. + Python object Raises: InsufficientDataException(UnpackException): @@ -977,10 +971,9 @@ def _unpack2(fp, **options): Duplicate key encountered during map unpacking. Example: - >>> f = open('test.bin', 'rb') - >>> umsgpack.unpackb(f) - {u'compact': True, u'schema': 0} - >>> + >>> f = open('test.bin', 'rb') + >>> umsgpack.unpackb(f) + {u'compact': True, u'schema': 0} """ return _unpack(fp, options) @@ -992,20 +985,20 @@ def _unpack3(fp, **options): Args: fp: a .read()-supporting file-like object - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext type to a callable that unpacks an instance of Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) use_tuple (bool): unpacks arrays into tuples, instead of lists (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) + :class:`InvalidString`, for access to the + bytes (default False) Returns: - A Python object. + Python object Raises: InsufficientDataException(UnpackException): @@ -1023,10 +1016,9 @@ def _unpack3(fp, **options): Duplicate key encountered during map unpacking. Example: - >>> f = open('test.bin', 'rb') - >>> umsgpack.unpackb(f) - {'compact': True, 'schema': 0} - >>> + >>> f = open('test.bin', 'rb') + >>> umsgpack.unpackb(f) + {'compact': True, 'schema': 0} """ return _unpack(fp, options) @@ -1037,22 +1029,22 @@ def _unpackb2(s, **options): Deserialize MessagePack bytes into a Python object. Args: - s: a 'str' or 'bytearray' containing serialized MessagePack bytes + s (str, bytearray): serialized MessagePack bytes - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext type to a callable that unpacks an instance of Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) use_tuple (bool): unpacks arrays into tuples, instead of lists (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) + :class:`InvalidString`, for access to the + bytes (default False) Returns: - A Python object. + Python object Raises: TypeError: @@ -1072,9 +1064,8 @@ def _unpackb2(s, **options): Duplicate key encountered during map unpacking. Example: - >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') - {u'compact': True, u'schema': 0} - >>> + >>> umsgpack.unpackb(b'\\x82\\xa7compact\\xc3\\xa6schema\\x00') + {u'compact': True, u'schema': 0} """ if not isinstance(s, (str, bytearray)): raise TypeError("packed data must be type 'str' or 'bytearray'") @@ -1087,22 +1078,22 @@ def _unpackb3(s, **options): Deserialize MessagePack bytes into a Python object. Args: - s: a 'bytes' or 'bytearray' containing serialized MessagePack bytes + s (bytes, bytearray): serialized MessagePack bytes - Kwargs: + Keyword Args: ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext type to a callable that unpacks an instance of Ext into an object - use_ordered_dict (bool): unpack maps into OrderedDict, instead of - unordered dict (default False) + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) use_tuple (bool): unpacks arrays into tuples, instead of lists (default False) allow_invalid_utf8 (bool): unpack invalid strings into instances of - InvalidString, for access to the bytes - (default False) + :class:`InvalidString`, for access to the + bytes (default False) Returns: - A Python object. + Python object Raises: TypeError: @@ -1122,9 +1113,8 @@ def _unpackb3(s, **options): Duplicate key encountered during map unpacking. Example: - >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') - {'compact': True, 'schema': 0} - >>> + >>> umsgpack.unpackb(b'\\x82\\xa7compact\\xc3\\xa6schema\\x00') + {'compact': True, 'schema': 0} """ if not isinstance(s, (bytes, bytearray)): raise TypeError("packed data must be type 'bytes' or 'bytearray'") From 18717a19f81c294d331942afed8ebcaaa3837653 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 10 May 2023 02:33:49 -0500 Subject: [PATCH 099/109] migrate documentation to sphinx --- .gitignore | 1 + README.md | 369 +---------------------------------------- docs/Makefile | 20 +++ docs/api.md | 105 ++++++++++++ docs/behavior-notes.md | 33 ++++ docs/conf.py | 47 ++++++ docs/examples.md | 117 +++++++++++++ docs/extension.md | 111 +++++++++++++ docs/index.md | 28 ++++ docs/installation.md | 19 +++ docs/license.md | 3 + docs/make.bat | 35 ++++ docs/packing.md | 103 ++++++++++++ docs/streaming.md | 54 ++++++ docs/unpacking.md | 225 +++++++++++++++++++++++++ msgpack.org.md | 2 +- 16 files changed, 910 insertions(+), 362 deletions(-) create mode 100644 docs/Makefile create mode 100644 docs/api.md create mode 100644 docs/behavior-notes.md create mode 100644 docs/conf.py create mode 100644 docs/examples.md create mode 100644 docs/extension.md create mode 100644 docs/index.md create mode 100644 docs/installation.md create mode 100644 docs/license.md create mode 100644 docs/make.bat create mode 100644 docs/packing.md create mode 100644 docs/streaming.md create mode 100644 docs/unpacking.md diff --git a/.gitignore b/.gitignore index 2b225f4..200d697 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ .tox/ .cache/ .pytest_cache/ +docs/_build diff --git a/README.md b/README.md index 24ebb63..82f7201 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) -u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. +u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. u-msgpack-python is currently distributed on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py). @@ -130,373 +130,20 @@ b'\x82\xa7compact\xc3\xa6schema\x00' >>> ``` -## Ext Serializable +## Documentation -The `ext_serializable()` decorator registers application classes for automatic -packing and unpacking with the specified Ext type. The decorator accepts the -Ext type code as an argument. The application class should implement a -`packb()` method that returns serialized bytes, and an `unpackb()` class method -or static method that accepts serialized bytes and returns an instance of the -application class. +Documentation is hosted at [https://u-msgpack-python.readthedocs.io](https://u-msgpack-python.readthedocs.io). -Example for registering, packing, and unpacking a custom class with Ext type -code 0x10: +To build documentation locally with Sphinx, run: -``` python -@umsgpack.ext_serializable(0x10) -class Point(object): - def __init__(self, x, y, z): - self.x = x - self.y = y - self.z = z - - def __str__(self): - return "Point({}, {}, {})".format(self.x, self.y, self.z) - - def packb(self): - return struct.pack(">iii", self.x, self.y, self.z) - - @staticmethod - def unpackb(data): - return Point(*struct.unpack(">iii", data)) - -# Pack -obj = Point(1,2,3) -data = umsgpack.packb(obj) - -# Unpack -obj = umsgpack.unpackb(data) -print(obj) # -> Point(1, 2, 3) -``` - -## Ext Handlers - -The packing functions accept an optional `ext_handlers` dictionary that maps -custom types to callables that pack the type into an Ext object. The callable -should accept the custom type object as an argument and return a packed -`umsgpack.Ext` object. - -Example for packing `set`, `complex`, and `decimal.Decimal` types into Ext -objects with type codes 0x20, 0x30, and 0x40, respectively: - -``` python ->>> umsgpack.packb([1, True, {"foo", 2}, complex(3, 4), decimal.Decimal("0.31")], -... ext_handlers = { -... set: lambda obj: umsgpack.Ext(0x20, umsgpack.packb(list(obj))), -... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), -... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), -... }) -b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31' ->>> -``` -Similarly, the unpacking functions accept an optional `ext_handlers` dictionary -that maps Ext type codes to callables that unpack the Ext into a custom object. -The callable should accept a `umsgpack.Ext` object as an argument and return an -unpacked custom type object. - -Example for unpacking Ext objects with type codes 0x20, 0x30, and 0x40, into -`set`, `complex`, and `decimal.Decimal` typed objects, respectively: - -``` python ->>> umsgpack.unpackb(b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31', -... ext_handlers = { -... 0x20: lambda ext: set(umsgpack.unpackb(ext.data)), -... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), -... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), -... }) -[1, True, {'foo', 2}, (3+4j), Decimal('0.31')] ->>> -``` - -Example for packing and unpacking a custom class: - -``` python -class Point(object): - def __init__(self, x, y, z): - self.x = x - self.y = y - self.z = z - - def __str__(self): - return "Point({}, {}, {})".format(self.x, self.y, self.z) - - def pack(self): - return struct.pack(">iii", self.x, self.y, self.z) - - @staticmethod - def unpack(data): - return Point(*struct.unpack(">iii", data)) - -# Pack -obj = Point(1,2,3) -data = umsgpack.packb(obj, ext_handlers = {Point: lambda obj: umsgpack.Ext(0x10, obj.pack())}) - -# Unpack -obj = umsgpack.unpackb(data, ext_handlers = {0x10: lambda ext: Point.unpack(ext.data)}) -print(obj) # -> Point(1, 2, 3) -``` - -## Streaming Serialization and Deserialization - -The streaming `pack()`/`dump()` and `unpack()`/`load()` functions allow packing and unpacking objects directly to and from a stream, respectively. Streaming may be necessary when unpacking serialized bytes whose size is unknown in advance, or it may be more convenient and efficient when working directly with stream objects (e.g. files or stream sockets). - -`pack(obj, fp)` / `dump(obj, fp)` serialize Python object `obj` to a `.write()` supporting file-like object `fp`. - -``` python ->>> class Foo: -... def write(self, data): -... # write 'data' bytes -... pass -... ->>> f = Foo() ->>> umsgpack.pack({u"compact": True, u"schema": 0}, f) ->>> -``` - -`unpack(fp)` / `load(fp)` deserialize a Python object from a `.read()` supporting file-like object `fp`. - -``` python ->>> class Bar: -... def read(self, n): -... # read and return 'n' number of bytes -... return b"\x01"*n -... ->>> f = Bar() ->>> umsgpack.unpack(f) -1 ->>> -``` - -## Options - -### Ordered Dictionaries - -The unpacking functions provide a `use_ordered_dict` option to unpack MessagePack maps into the `collections.OrderedDict` type, rather than the unordered `dict` type, to preserve the order of deserialized MessagePack maps. - -``` python ->>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') -{'compact': True, 'schema': 0} ->>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00', use_ordered_dict=True) -OrderedDict([('compact', True), ('schema', 0)]) ->>> ``` - -## Tuples - -The unpacking functions provide a `use_tuple` option to unpack MessagePack arrays into tuples, rather than lists. - -``` python ->>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03') -['a', True, [1, [2, 3]]] ->>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03', use_tuple=True) -('a', True, (1, (2, 3))) ->>> +cd docs +make html ``` -### Invalid UTF-8 Strings - -The unpacking functions provide an `allow_invalid_utf8` option to unpack MessagePack strings with invalid UTF-8 into the `umsgpack.InvalidString` type, instead of throwing an exception. The `umsgpack.InvalidString` type is a subclass of `bytes`, and can be used like any other `bytes` object. - -``` python ->>> # Attempt to unpack invalid UTF-8 string -... umsgpack.unpackb(b'\xa4\x80\x01\x02\x03') -... -umsgpack.InvalidStringException: unpacked string is invalid utf-8 ->>> umsgpack.unpackb(b'\xa4\x80\x01\x02\x03', allow_invalid_utf8=True) -b'\x80\x01\x02\x03' ->>> -``` - -### Float Precision - -The packing functions provide a `force_float_precision` option to force packing of floats into the specified precision: `"single"` for IEEE-754 single-precision floats, or `"double"` for IEEE-754 double-precision floats. - -``` python ->>> # Force float packing to single-precision floats -... umsgpack.packb(2.5, force_float_precision="single") -b'\xca@ \x00\x00' ->>> # Force float packing to double-precision floats -... umsgpack.packb(2.5, force_float_precision="double") -b'\xcb@\x04\x00\x00\x00\x00\x00\x00' ->>> -``` - -### Old Specification Compatibility Mode - -The compatibility mode supports the "raw" bytes MessagePack type from the [old specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md). When the module-wide `compatibility` option is enabled, both unicode strings and bytes will be serialized into the "raw" MessagePack type, and the "raw" MessagePack type will be deserialized into bytes. - -``` python ->>> umsgpack.compatibility = True ->>> ->>> umsgpack.packb([u"some string", b"some bytes"]) -b'\x92\xabsome string\xaasome bytes' ->>> umsgpack.unpackb(_) -[b'some string', b'some bytes'] ->>> -``` - -## Exceptions - -### Packing Exceptions - -If an error occurs during packing, umsgpack will raise an exception derived from `umsgpack.PackException`. All possible packing exceptions are described below. - -* `UnsupportedTypeException`: Object type not supported for packing. - - ``` python - >>> # Attempt to pack set type - ... umsgpack.packb(set([1,2,3])) - ... - umsgpack.UnsupportedTypeException: unsupported type: - >>> - - >>> # Attempt to pack > 64-bit unsigned int - ... umsgpack.packb(2**64) - ... - umsgpack.UnsupportedTypeException: huge unsigned int - >>> - ``` - -* `NotImplementedError`: Ext serializable class is missing implementation of `packb()`. - - ``` python - >>> @umsgpack.ext_serializable(0x50) - ... class Point(collections.namedtuple('Point', ['x', 'y'])): - ... pass - ... - >>> umsgpack.packb(Point(1, 2)) - ... - NotImplementedError: Ext serializable class is missing implementation of packb() - >>> - ``` - -### Unpacking Exceptions - -If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a `TypeError` exception. If an error occurs during unpacking, umsgpack will raise an exception derived from `umsgpack.UnpackException`. All possible unpacking exceptions are described below. - -* `TypeError`: Packed data is not type `str` (Python 2), or not type `bytes` (Python 3). - - ``` python - # Attempt to unpack non-str type data in Python 2 - >>> umsgpack.unpackb(u"no good") - ... - TypeError: expected packed data as type 'str' - >>> - - # Attempt to unpack non-bytes type data in Python 3 - >>> umsgpack.unpackb("no good") - ... - TypeError: expected packed data as type 'bytes' - >>> - ``` - -* `InsufficientDataException`: Insufficient data to unpack the serialized object. - - ``` python - # Attempt to unpack a cut-off serialized 32-bit unsigned int - >>> umsgpack.unpackb(b"\xce\xff\xff\xff") - ... - umsgpack.InsufficientDataException - >>> - - # Attempt to unpack an array of length 2 missing the second item - >>> umsgpack.unpackb(b"\x92\xc2") - ... - umsgpack.InsufficientDataException - >>> - - ``` -* `InvalidStringException`: Invalid UTF-8 string encountered during unpacking. - - String bytes are strictly decoded with UTF-8. This exception is thrown if - UTF-8 decoding of string bytes fails. Use the `allow_invalid_utf8` option - to unpack invalid MessagePack strings into byte strings. - - ``` python - # Attempt to unpack invalid UTF-8 string - >>> umsgpack.unpackb(b"\xa2\x80\x81") - ... - umsgpack.InvalidStringException: unpacked string is invalid utf-8 - >>> - ``` - -* `UnsupportedTimestampException`: Unsupported timestamp encountered during unpacking. - - The official timestamp extension type supports 32-bit, 64-bit and 96-bit - formats. This exception is thrown if a timestamp extension type with an - unsupported format is encountered. - - ``` python - # Attempt to unpack invalid timestamp - >>> umsgpack.unpackb(b"\xd5\xff\x01\x02") - ... - umsgpack.UnsupportedTimestampException: unsupported timestamp with data length 2 - >>> - ``` - -* `ReservedCodeException`: Reserved code encountered during unpacking. - - ``` python - # Attempt to unpack reserved code 0xc1 - >>> umsgpack.unpackb(b"\xc1") - ... - umsgpack.ReservedCodeException: reserved code encountered: 0xc1 - >>> - ``` - -* `UnhashableKeyException`: Unhashable key encountered during map unpacking. The packed map cannot be unpacked into a Python dictionary. - - Python dictionaries only support keys that are instances of `collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a MessagePack serialization, it cannot be unpacked into a valid Python dictionary. - - ``` python - # Attempt to unpack { {} : False } - >>> umsgpack.unpackb(b"\x82\x80\xc2") - ... - umsgpack.UnhashableKeyException: encountered unhashable key type: {}, - >>> - ``` - -* `DuplicateKeyException`: Duplicate key encountered during map unpacking. - - Python dictionaries do not support duplicate keys, but MessagePack maps may be serialized with duplicate keys. - - ``` python - # Attempt to unpack { 1: True, 1: False } - >>> umsgpack.unpackb(b"\x82\x01\xc3\x01\xc2") - ... - umsgpack.DuplicateKeyException: encountered duplicate key: 1, - >>> - ``` - -* `NotImplementedError`: Ext serializable class is missing implementation of `unpackb()`. - - ``` python - >>> @umsgpack.ext_serializable(0x50) - ... class Point(collections.namedtuple('Point', ['x', 'y'])): - ... pass - ... - >>> umsgpack.unpackb(b'\xd7\x50\x00\x00\x00\x01\x00\x00\x00\x02') - ... - NotImplementedError: Ext serializable class is missing implementation of unpackb() - >>> - ``` - -## Behavior Notes +Sphinx will produce the HTML documentation in `docs/_build/html/`. -* Python 2 - * `unicode` type objects are packed into, and unpacked from, the msgpack `string` format - * `str` type objects are packed into, and unpacked from, the msgpack `binary` format -* Python 3 - * `str` type objects are packed into, and unpacked from, the msgpack `string` format - * `bytes` type objects are packed into, and unpacked from, the msgpack `binary` format -* The msgpack string format is strictly decoded with UTF-8 — an exception is thrown if the string bytes cannot be decoded into a valid UTF-8 string, unless the `allow_invalid_utf8` option is enabled -* The msgpack array format is unpacked into a Python list, unless it is the key of a map, in which case it is unpacked into a Python tuple -* Python tuples and lists are both packed into the msgpack array format -* Python float types are packed into the msgpack float32 or float64 format depending on the system's `sys.float_info` -* The Python `datetime.datetime` type is packed into, and unpacked from, the msgpack `timestamp` format - * Note that this Python type only supports microsecond resolution, while the msgpack `timestamp` format supports nanosecond resolution. Timestamps with finer than microsecond resolution will lose precision during unpacking. Users may override the packing and unpacking of the msgpack `timestamp` format with a custom type for alternate behavior. - * Both naive and aware timestamp are supported. Naive timestamps are packed as if they are in the UTC timezone. Timestamps are always unpacked as aware `datetime.datetime` objects in the UTC timezone. -* Ext type handlers specified in the optional `ext_handlers` dictionary will override `ext_serializable()` classes during packing and unpacking +Run `make help` to see other output targets (LaTeX, man, text, etc.). ## Testing diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..d53f6d4 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,105 @@ +# API + +## Packing + +```{eval-rst} +.. autofunction:: umsgpack.packb + +Also available under the ``umsgpack.dumps()`` alias. +``` + +```{eval-rst} +.. autofunction:: umsgpack.pack + +Also available under the ``umsgpack.dump()`` alias. +``` + +## Unpacking + +```{eval-rst} +.. autofunction:: umsgpack.unpackb + +Also available under the ``umsgpack.loads()`` alias. +``` + +```{eval-rst} +.. autofunction:: umsgpack.unpack + +Also available under the ``umsgpack.load()`` alias. +``` + +## Packing Exceptions + +```{eval-rst} +.. autoexception:: umsgpack.PackException +``` + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTypeException +``` + +## Unpacking Exceptions + +```{eval-rst} +.. autoexception:: umsgpack.UnpackException +``` + +```{eval-rst} +.. autoexception:: umsgpack.InsufficientDataException +``` + +```{eval-rst} +.. autoexception:: umsgpack.InvalidStringException +``` + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTimestampException +``` + +```{eval-rst} +.. autoexception:: umsgpack.ReservedCodeException +``` + +```{eval-rst} +.. autoexception:: umsgpack.UnhashableKeyException +``` + +```{eval-rst} +.. autoexception:: umsgpack.DuplicateKeyException +``` + +## Ext Class + +```{eval-rst} +.. autoclass:: umsgpack.Ext + :member-order: bysource + :special-members: __init__, __eq__, __ne__, __str__, __hash__ +``` + +## Ext Serializable Decorator + +```{eval-rst} +.. autodecorator:: umsgpack.ext_serializable +``` + +## Invalid String Class + +```{eval-rst} +.. autoclass:: umsgpack.InvalidString +``` + +## Attributes + +```{eval-rst} +.. autodata:: umsgpack.compatibility +``` + +## Constants + +```{eval-rst} +.. autodata:: umsgpack.version +``` + +```{eval-rst} +.. autodata:: umsgpack.__version__ +``` diff --git a/docs/behavior-notes.md b/docs/behavior-notes.md new file mode 100644 index 0000000..25474ee --- /dev/null +++ b/docs/behavior-notes.md @@ -0,0 +1,33 @@ +# Behavior Notes + +* Python 2 + * `unicode` type objects are packed into, and unpacked from, the MessagePack + `string` format + * `str` type objects are packed into, and unpacked from, the MessagePack + `binary` format +* Python 3 + * `str` type objects are packed into, and unpacked from, the MessagePack + `string` format + * `bytes` type objects are packed into, and unpacked from, the MessagePack + `binary` format +* The MessagePack string format is strictly decoded with UTF-8 — an exception + is thrown if the string bytes cannot be decoded into a valid UTF-8 string, + unless the `allow_invalid_utf8` option is enabled +* The MessagePack array format is unpacked into a Python list, unless it is the + key of a map, in which case it is unpacked into a Python tuple +* Python tuples and lists are both packed into the MessagePack array format +* Python float types are packed into the MessagePack float32 or float64 format + depending on the system's `sys.float_info` +* The Python `datetime.datetime` type is packed into, and unpacked from, the + MessagePack `timestamp` format + * Note that the Python `datetime.datetime` type only supports microsecond + resolution, while the MessagePack `timestamp` format supports nanosecond + resolution. Timestamps with finer than microsecond resolution will lose + precision during unpacking. Users may override the packing and unpacking + of the MessagePack `timestamp` format with a custom type for alternate + behavior. + * Both naive and aware timestamp are supported. Naive timestamps are packed + as if they are in the UTC timezone. Timestamps are always unpacked as + aware `datetime.datetime` objects in the UTC timezone. +* Ext type handlers specified in the optional `ext_handlers` dictionary will + override `ext_serializable()` classes during packing and unpacking diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..6856870 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,47 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +import umsgpack + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'u-msgpack-python' +copyright = '2013-2023, Vanya A. Sergeev' +author = 'Vanya A. Sergeev' +release = '2.7.2' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'sphinx_rtd_theme', + 'myst_parser', +] + +templates_path = ['_templates'] +exclude_patterns = ['_build'] + +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] +html_show_sourcelink = False diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..6d63f8c --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,117 @@ +# Examples + +Basic Example: + +``` python +>>> import umsgpack +>>> umsgpack.packb({u"compact": True, u"schema": 0}) +b'\x82\xa7compact\xc3\xa6schema\x00' +>>> umsgpack.unpackb(_) +{u'compact': True, u'schema': 0} +>>> +``` + +A more complicated example: + +``` python +>>> umsgpack.packb([1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", \ +... u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345]) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' +>>> umsgpack.unpackb(_) +[1, True, False, 4294967295, {u'foo': b'\x80\x01\x02', \ + u'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] +>>> +``` + +Streaming serialization with file-like objects: + +``` python +>>> f = open('test.bin', 'wb') +>>> umsgpack.pack({u"compact": True, u"schema": 0}, f) +>>> umsgpack.pack([1,2,3], f) +>>> f.close() +>>> +>>> f = open('test.bin', 'rb') +>>> umsgpack.unpack(f) +{u'compact': True, u'schema': 0} +>>> umsgpack.unpack(f) +[1, 2, 3] +>>> f.close() +>>> +``` + +Serializing and deserializing a raw Ext type: + +``` python +>>> # Create an Ext object with type 5 and data b"\x01\x02\x03" +... foo = umsgpack.Ext(5, b"\x01\x02\x03") +>>> umsgpack.packb({u"stuff": foo, u"awesome": True}) +b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> +>>> bar = umsgpack.unpackb(_) +>>> print(bar['stuff']) +Ext Object (Type: 5, Data: 0x01 0x02 0x03) +>>> bar['stuff'].type +5 +>>> bar['stuff'].data +b'\x01\x02\x03' +>>> +``` + +Serializing and deserializing application-defined types with + +`ext_serializable()`: +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... def packb(self): +... return struct.pack(">ii", self.x, self.y) +... @staticmethod +... def unpackb(data): +... return Point(*struct.unpack(">ii", data)) +... +>>> umsgpack.packb(Point(1, 2)) +b'\xd7P\x00\x00\x00\x01\x00\x00\x00\x02' +>>> umsgpack.unpackb(_) +Point(x=1, y=2) +>>> +``` + +Serializing and deserializing application-defined types with Ext handlers: + +``` python +>>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], +... ext_handlers = { +... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xd6@0.31' +>>> umsgpack.unpackb(_, +... ext_handlers = { +... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), +... }) +[(1+2j), Decimal('0.31')] +>>> +``` + +Python standard library style names `dump`, `dumps`, `load`, `loads` are also +available: + +``` python +>>> umsgpack.dumps({u"compact": True, u"schema": 0}) +b'\x82\xa7compact\xc3\xa6schema\x00' +>>> umsgpack.loads(_) +{u'compact': True, u'schema': 0} +>>> +>>> f = open('test.bin', 'wb') +>>> umsgpack.dump({u"compact": True, u"schema": 0}, f) +>>> f.close() +>>> +>>> f = open('test.bin', 'rb') +>>> umsgpack.load(f) +{u'compact': True, u'schema': 0} +>>> +``` diff --git a/docs/extension.md b/docs/extension.md new file mode 100644 index 0000000..c967809 --- /dev/null +++ b/docs/extension.md @@ -0,0 +1,111 @@ +# Extension Types + +u-msgpack-python supports two mechanisms for packing and unpacking MessagePack +Ext types: the `ext_handlers` keyword option, and the `ext_serializable()` +decorator. + +## Ext Handlers + +The packing functions accept an optional `ext_handlers` dictionary that maps +custom types to callables that pack the type into an Ext object. The callable +should accept the custom type object as an argument and return a packed +`umsgpack.Ext` object. + +Example for packing `set`, `complex`, and `decimal.Decimal` types into Ext +objects with type codes 0x20, 0x30, and 0x40, respectively: + +``` python +>>> umsgpack.packb([1, True, {"foo", 2}, complex(3, 4), decimal.Decimal("0.31")], +... ext_handlers = { +... set: lambda obj: umsgpack.Ext(0x20, umsgpack.packb(list(obj))), +... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31' +>>> +``` +Similarly, the unpacking functions accept an optional `ext_handlers` dictionary +that maps Ext type codes to callables that unpack the Ext into a custom object. +The callable should accept a `umsgpack.Ext` object as an argument and return an +unpacked custom type object. + +Example for unpacking Ext objects with type codes 0x20, 0x30, and 0x40, into +`set`, `complex`, and `decimal.Decimal` typed objects, respectively: + +``` python +>>> umsgpack.unpackb(b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31', +... ext_handlers = { +... 0x20: lambda ext: set(umsgpack.unpackb(ext.data)), +... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), +... }) +[1, True, {'foo', 2}, (3+4j), Decimal('0.31')] +>>> +``` + +Example for packing and unpacking a custom class: + +``` python +class Point(object): + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z + + def __str__(self): + return "Point({}, {}, {})".format(self.x, self.y, self.z) + + def pack(self): + return struct.pack(">iii", self.x, self.y, self.z) + + @staticmethod + def unpack(data): + return Point(*struct.unpack(">iii", data)) + +# Pack +obj = Point(1,2,3) +data = umsgpack.packb(obj, ext_handlers = {Point: lambda obj: umsgpack.Ext(0x10, obj.pack())}) + +# Unpack +obj = umsgpack.unpackb(data, ext_handlers = {0x10: lambda ext: Point.unpack(ext.data)}) +print(obj) # -> Point(1, 2, 3) +``` + +## Ext Serializable + +The `ext_serializable()` decorator registers application classes for automatic +packing and unpacking with the specified Ext type. The decorator accepts the +Ext type code as an argument. The application class should implement a +`packb()` method that returns serialized bytes, and an `unpackb()` class method +or static method that accepts serialized bytes and returns an instance of the +application class. + +Example for registering, packing, and unpacking a custom class with Ext type +code 0x10: + +``` python +@umsgpack.ext_serializable(0x10) +class Point(object): + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z + + def __str__(self): + return "Point({}, {}, {})".format(self.x, self.y, self.z) + + def packb(self): + return struct.pack(">iii", self.x, self.y, self.z) + + @staticmethod + def unpackb(data): + return Point(*struct.unpack(">iii", data)) + +# Pack +obj = Point(1,2,3) +data = umsgpack.packb(obj) + +# Unpack +obj = umsgpack.unpackb(data) +print(obj) # -> Point(1, 2, 3) +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..0c39881 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,28 @@ +# Welcome to u-msgpack-python's documentation! + +[u-msgpack-python](https://github.com/vsergeev/u-msgpack-python) is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. + +u-msgpack-python is currently distributed on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py). + +## Contents + + +```{toctree} +:maxdepth: 1 + +Home +installation.md +examples.md +packing.md +unpacking.md +streaming.md +extension.md +api.md +behavior-notes.md +license.md +``` + +## Indices and tables + +* [](genindex) +* [](search) diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..3da50a3 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,19 @@ +# Installation + +Install with pip: + +``` text +$ pip install u-msgpack-python +``` + +Install with easy_install: + +``` text +$ easy_install u-msgpack-python +``` + +or simply drop [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) into your project! + +``` text +$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py +``` diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..9594f3e --- /dev/null +++ b/docs/license.md @@ -0,0 +1,3 @@ +# License + +u-msgpack-python is MIT licensed. See the included [`LICENSE`](https://raw.github.com/vsergeev/u-msgpack-python/master/LICENSE) file for more details. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/packing.md b/docs/packing.md new file mode 100644 index 0000000..b5293db --- /dev/null +++ b/docs/packing.md @@ -0,0 +1,103 @@ +# Packing + +## Example + +``` python +>>> umsgpack.packb([1, True, False, 0xffffffff, {'foo': b'\x80\x01\x02', \ +... 'bar': [1,2,3, {'a': [1,2,3,{}]}]}, -1, 2.12345]) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' +>>> +``` + +## API + +```{eval-rst} +.. autofunction:: umsgpack.packb + :noindex: + +Also available under the ``umsgpack.dumps()`` alias. +``` + +## Options + +### Ext Handlers + +See the [Extension Types](extension.md) section. + +### Float Precision + +The packing functions provide a `force_float_precision` option to force packing +of floats into the specified precision: `"single"` for IEEE-754 +single-precision floats, or `"double"` for IEEE-754 double-precision floats. + +``` python +>>> # Force float packing to single-precision floats +... umsgpack.packb(2.5, force_float_precision="single") +b'\xca@ \x00\x00' +>>> # Force float packing to double-precision floats +... umsgpack.packb(2.5, force_float_precision="double") +b'\xcb@\x04\x00\x00\x00\x00\x00\x00' +>>> +``` + +### Old Specification Compatibility Mode + +The compatibility mode supports the "raw" bytes MessagePack type from the [old +specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md). +When the module-wide `compatibility` attribute is enabled, both unicode strings +and bytes will be serialized into the "raw" MessagePack type, and the "raw" +MessagePack type will be deserialized into bytes. + +``` python +>>> umsgpack.compatibility = True +>>> +>>> umsgpack.packb([u"some string", b"some bytes"]) +b'\x92\xabsome string\xaasome bytes' +>>> umsgpack.unpackb(_) +[b'some string', b'some bytes'] +>>> +``` + +## Exceptions + +If an error occurs during packing, u-msgpack-python will raise an exception +derived from `umsgpack.PackException`. Possible packing exceptions are +described below. + +### UnsupportedTypeException + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTypeException + :noindex: +``` + +``` python +>>> # Attempt to pack set type +... umsgpack.packb(set([1,2,3])) +... +umsgpack.UnsupportedTypeException: unsupported type: +>>> + +>>> # Attempt to pack > 64-bit unsigned int +... umsgpack.packb(2**64) +... +umsgpack.UnsupportedTypeException: huge unsigned int +>>> +``` + +### NotImplementedError + +Ext serializable class is missing implementation of `packb()`. + +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... pass +... +>>> umsgpack.packb(Point(1, 2)) +... +NotImplementedError: Ext serializable class is missing implementation of packb() +>>> +``` diff --git a/docs/streaming.md b/docs/streaming.md new file mode 100644 index 0000000..2e30fdc --- /dev/null +++ b/docs/streaming.md @@ -0,0 +1,54 @@ +# Streaming + +The streaming `pack()` and `unpack()` functions allow packing and unpacking +objects directly to and from a stream, respectively. Streaming may be necessary +when unpacking serialized bytes whose size is unknown in advance, or it may be +more convenient and efficient when working directly with stream objects (e.g. +files or stream sockets). + +## Packing + +`pack(obj, fp)` serializes Python object `obj` to a `.write()` supporting +file-like object `fp`. + +``` python +>>> class Foo: +... def write(self, data): +... # write 'data' bytes +... pass +... +>>> f = Foo() +>>> umsgpack.pack({u"compact": True, u"schema": 0}, f) +>>> +``` + +```{eval-rst} +.. autofunction:: umsgpack.pack + :noindex: + +Also available under the ``umsgpack.dump()`` alias. +``` + +## Unpacking + +`unpack(fp)` deserializes a Python object from a `.read()` supporting file-like +object `fp`. + +``` python +>>> class Bar: +... def read(self, n): +... # read and return 'n' number of bytes +... return b"\x01"*n +... +>>> f = Bar() +>>> umsgpack.unpack(f) +1 +>>> +``` + +```{eval-rst} +.. autofunction:: umsgpack.unpack + :noindex: + +Also available under the ``umsgpack.load()`` alias. +``` diff --git a/docs/unpacking.md b/docs/unpacking.md new file mode 100644 index 0000000..43ccf94 --- /dev/null +++ b/docs/unpacking.md @@ -0,0 +1,225 @@ +# Unpacking + +## Example + +``` python +>>> umsgpack.unpackb(b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3\ +foo\xc4\x03\x80\x01\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\ +\x03\x80\xff\xcb@\x00\xfc\xd3Z\x85\x87\x94') +[1, True, False, 4294967295, {'foo': b'\x80\x01\x02', \ + 'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] +>>> +``` + +## API + +```{eval-rst} +.. autofunction:: umsgpack.unpackb + :noindex: + +Also available under the ``umsgpack.loads()`` alias. +``` + +## Options + +### Ext Handlers + +See the [Extension Types](extension.md) section. + +### Ordered Dictionaries + +The unpacking functions provide a `use_ordered_dict` option to unpack +MessagePack maps into the `collections.OrderedDict` type, rather than the +unordered `dict` type, to preserve the order of deserialized MessagePack maps. +Note that as of Python 3.6, dictionaries are insertion ordered by default. + +``` python +>>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') +{'compact': True, 'schema': 0} +>>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00', use_ordered_dict=True) +OrderedDict([('compact', True), ('schema', 0)]) +>>> +``` + +### Tuples + +The unpacking functions provide a `use_tuple` option to unpack MessagePack +arrays into tuples, rather than lists. + +``` python +>>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03') +['a', True, [1, [2, 3]]] +>>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03', use_tuple=True) +('a', True, (1, (2, 3))) +>>> +``` + +### Invalid UTF-8 Strings + +The unpacking functions provide an `allow_invalid_utf8` option to unpack +MessagePack strings with invalid UTF-8 into the `umsgpack.InvalidString` type, +instead of throwing an exception. The `umsgpack.InvalidString` type is a +subclass of `bytes`, and can be used like any other `bytes` object. + +``` python +>>> # Attempt to unpack invalid UTF-8 string +... umsgpack.unpackb(b'\xa4\x80\x01\x02\x03') +... +umsgpack.InvalidStringException: unpacked string is invalid utf-8 +>>> umsgpack.unpackb(b'\xa4\x80\x01\x02\x03', allow_invalid_utf8=True) +b'\x80\x01\x02\x03' +>>> +``` + +## Exceptions + +If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise +a `TypeError` exception. If an error occurs during unpacking, u-msgpack-python +will raise an exception derived from `umsgpack.UnpackException`. Possible +unpacking exceptions are described below. + + +### TypeError + +Packed data is not type `str` (Python 2), or not type `bytes` (Python 3). + +``` python +# Attempt to unpack non-str type data in Python 2 +>>> umsgpack.unpackb(u"no good") +... +TypeError: expected packed data as type 'str' +>>> + +# Attempt to unpack non-bytes type data in Python 3 +>>> umsgpack.unpackb("no good") +... +TypeError: expected packed data as type 'bytes' +>>> +``` + +### InsufficientDataException + +```{eval-rst} +.. autoexception:: umsgpack.InsufficientDataException + :noindex: +``` + +``` python +# Attempt to unpack a cut-off serialized 32-bit unsigned int +>>> umsgpack.unpackb(b"\xce\xff\xff\xff") +... +umsgpack.InsufficientDataException +>>> + +# Attempt to unpack an array of length 2 missing the second item +>>> umsgpack.unpackb(b"\x92\xc2") +... +umsgpack.InsufficientDataException +>>> +``` + +### InvalidStringException + +```{eval-rst} +.. autoexception:: umsgpack.InvalidStringException + :noindex: +``` + +String bytes are strictly decoded with UTF-8. This exception is thrown if UTF-8 +decoding of string bytes fails. Use the `allow_invalid_utf8` option to unpack +invalid MessagePack strings into byte strings. + +``` python +# Attempt to unpack invalid UTF-8 string +>>> umsgpack.unpackb(b"\xa2\x80\x81") +... +umsgpack.InvalidStringException: unpacked string is invalid utf-8 +>>> +``` + +### UnsupportedTimestampException + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTimestampException + :noindex: +``` + +The official timestamp extension type supports 32-bit, 64-bit and 96-bit +formats. This exception is thrown if a timestamp extension type with an +unsupported format is encountered. + +``` python +# Attempt to unpack invalid timestamp +>>> umsgpack.unpackb(b"\xd5\xff\x01\x02") +... +umsgpack.UnsupportedTimestampException: unsupported timestamp with data length 2 +>>> +``` + +### ReservedCodeException + +```{eval-rst} +.. autoexception:: umsgpack.ReservedCodeException + :noindex: +``` + +``` python +# Attempt to unpack reserved code 0xc1 +>>> umsgpack.unpackb(b"\xc1") +... +umsgpack.ReservedCodeException: reserved code encountered: 0xc1 +>>> +``` + +### UnhashableKeyException + +```{eval-rst} +.. autoexception:: umsgpack.UnhashableKeyException + :noindex: +``` + +Python dictionaries only support keys that are instances of +`collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a +MessagePack serialization, it cannot be unpacked into a valid Python +dictionary. + +``` python +# Attempt to unpack { {} : False } +>>> umsgpack.unpackb(b"\x82\x80\xc2") +... +umsgpack.UnhashableKeyException: encountered unhashable key type: {}, +>>> +``` + +### DuplicateKeyException + +```{eval-rst} +.. autoexception:: umsgpack.DuplicateKeyException + :noindex: +``` + +Python dictionaries do not support duplicate keys, but MessagePack maps may be +serialized with duplicate keys. + +``` python +# Attempt to unpack { 1: True, 1: False } +>>> umsgpack.unpackb(b"\x82\x01\xc3\x01\xc2") +... +umsgpack.DuplicateKeyException: encountered duplicate key: 1, +>>> +``` + +### NotImplementedError + +Ext serializable class is missing implementation of `unpackb()`. + +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... pass +... +>>> umsgpack.unpackb(b'\xd7\x50\x00\x00\x00\x01\x00\x00\x00\x02') +... +NotImplementedError: Ext serializable class is missing implementation of unpackb() +>>> +``` diff --git a/msgpack.org.md b/msgpack.org.md index ae4c100..a832e26 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -1,6 +1,6 @@ # u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) -u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). +u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). u-msgpack-python is currently distributed on PyPI: https://pypi.python.org/pypi/u-msgpack-python and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) From 47e19a2f1bd5573fff76bf6ea816e0dd709cbda5 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 12 May 2023 01:16:51 -0500 Subject: [PATCH 100/109] add readthedocs configuration --- .readthedocs.yaml | 13 +++++++++++++ docs/requirements.txt | 2 ++ 2 files changed, 15 insertions(+) create mode 100644 .readthedocs.yaml create mode 100644 docs/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..f334b70 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..7c98d98 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx-rtd-theme>=1.2.0 +myst-parser From 9d67b976a085a03cfdd8306222fb569a96683956 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Wed, 10 May 2023 02:35:08 -0500 Subject: [PATCH 101/109] add docs badge to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 82f7201..d6cdda7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) +# u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![Docs Status](https://readthedocs.org/projects/u-msgpack-python/badge/)](https://u-msgpack-python.readthedocs.io/en/latest/) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. From 98ad62fda6819fbb4c23251395576b471f32d4d2 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Fri, 12 May 2023 00:49:49 -0500 Subject: [PATCH 102/109] add type stubs --- umsgpack.pyi | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 umsgpack.pyi diff --git a/umsgpack.pyi b/umsgpack.pyi new file mode 100644 index 0000000..61a8413 --- /dev/null +++ b/umsgpack.pyi @@ -0,0 +1,41 @@ +from typing import Any + +__version__: str + +version: tuple[int, int, int] + +def pack(obj, fp, **options) -> None: ... +def packb(obj, **options) -> bytes: ... +def dump(obj, fp, **options) -> None: ... +def dumps(obj, **options) -> bytes: ... + +def unpackb(s: bytes | bytearray, **options) -> Any: ... +def unpack(fp, **options) -> Any: ... +def loads(s: bytes | bytearray, **options) -> Any: ... +def load(fp, **options) -> Any: ... + +class Ext: + type: int + data: bytes + def __init__(self, type: int, data: bytes) -> None: ... + def __eq__(self, other) -> bool: ... + def __ne__(self, other) -> bool: ... + def __hash__(self) -> int: ... + +class InvalidString(bytes): ... + +def ext_serializable(ext_type: int): ... + +class PackException(Exception): ... +class UnpackException(Exception): ... +class UnsupportedTypeException(PackException): ... +class InsufficientDataException(UnpackException): ... +class InvalidStringException(UnpackException): ... +class UnsupportedTimestampException(UnpackException): ... +class ReservedCodeException(UnpackException): ... +class UnhashableKeyException(UnpackException): ... +class DuplicateKeyException(UnpackException): ... +KeyNotPrimitiveException = UnhashableKeyException +KeyDuplicateException = DuplicateKeyException + +compatibility: bool From 2ea8b78d63e39603acaf1aad8683e9690efb6711 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 16 May 2023 02:03:16 -0500 Subject: [PATCH 103/109] migrate from module to package --- README.md | 6 +++--- docs/index.md | 2 +- docs/installation.md | 4 ++-- msgpack.org.md | 6 +++--- setup.py | 3 ++- umsgpack.py => umsgpack/__init__.py | 0 umsgpack.pyi => umsgpack/__init__.pyi | 0 7 files changed, 11 insertions(+), 10 deletions(-) rename umsgpack.py => umsgpack/__init__.py (100%) rename umsgpack.pyi => umsgpack/__init__.pyi (100%) diff --git a/README.md b/README.md index d6cdda7..4d40466 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. -u-msgpack-python is currently distributed on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py). +u-msgpack-python is currently distributed as a package on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file module. ## Installation @@ -16,9 +16,9 @@ With easy_install: $ easy_install u-msgpack-python ``` -or simply drop [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) into your project! +or simply drop `umsgpack.py` into your project! ``` text -$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py +$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack/__init__.py -O umsgpack.py ``` ## Examples diff --git a/docs/index.md b/docs/index.md index 0c39881..f18f8ff 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,7 +2,7 @@ [u-msgpack-python](https://github.com/vsergeev/u-msgpack-python) is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. -u-msgpack-python is currently distributed on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py). +u-msgpack-python is currently distributed as a package on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file module. ## Contents diff --git a/docs/installation.md b/docs/installation.md index 3da50a3..8651fd6 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -12,8 +12,8 @@ Install with easy_install: $ easy_install u-msgpack-python ``` -or simply drop [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) into your project! +or simply drop `umsgpack.py` into your project! ``` text -$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py +$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack/__init__.py -O umsgpack.py ``` diff --git a/msgpack.org.md b/msgpack.org.md index a832e26..d508bf0 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -2,7 +2,7 @@ u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). -u-msgpack-python is currently distributed on PyPI: https://pypi.python.org/pypi/u-msgpack-python and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) +u-msgpack-python is currently distributed as a package on PyPI: https://pypi.python.org/pypi/u-msgpack-python and as a single file module. ## Installation @@ -16,9 +16,9 @@ With easy_install: $ easy_install u-msgpack-python ``` -or simply drop [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) into your project! +or simply drop `umsgpack.py` into your project! ``` text -$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py +$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack/__init__.py -O umsgpack.py ``` ## Examples diff --git a/setup.py b/setup.py index ced0b12..5d5ecc3 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,8 @@ author='vsergeev', author_email='v@sergeev.io', url='https://github.com/vsergeev/u-msgpack-python', - py_modules=['umsgpack'], + packages=['umsgpack'], + package_data={'umsgpack': ['*.pyi', 'py.typed']}, long_description="""u-msgpack-python is a lightweight `MessagePack `_ serializer and deserializer module written in pure Python, compatible with both Python 2 and Python 3, as well as CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest `MessagePack specification `_. In particular, it supports the new binary, UTF-8 string, and application-defined ext types. See https://github.com/vsergeev/u-msgpack-python for more information.""", classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/umsgpack.py b/umsgpack/__init__.py similarity index 100% rename from umsgpack.py rename to umsgpack/__init__.py diff --git a/umsgpack.pyi b/umsgpack/__init__.pyi similarity index 100% rename from umsgpack.pyi rename to umsgpack/__init__.pyi From e5c9b9b16ae9b5a482aa11223c43574927fc163d Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Thu, 18 May 2023 04:18:42 -0500 Subject: [PATCH 104/109] use pypy2 environment in tox config instead of pypy --- .travis.yml | 2 +- tox.ini | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e295bf7..5c0a602 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,6 @@ matrix: - python: 3.11 env: TOXENV=py311 - python: pypy2.7-7.3.1 - env: TOXENV=pypy + env: TOXENV=pypy2 - python: pypy3.6-7.3.1 env: TOXENV=pypy3 diff --git a/tox.ini b/tox.ini index 5499181..7f0edc0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27, py35, py36, py37, py38, py39, py310, py311, pypy, pypy3 +envlist = py27, py35, py36, py37, py38, py39, py310, py311, pypy2, pypy3 skip_missing_interpreters=true [testenv] deps = pytest From 5ba07ec23457839e27020ff3441fdfd2c6957d87 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Thu, 18 May 2023 03:54:45 -0500 Subject: [PATCH 105/109] add mypy environment to tox config --- tox.ini | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 7f0edc0..6b6086b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,11 @@ [tox] -envlist = py27, py35, py36, py37, py38, py39, py310, py311, pypy2, pypy3 +envlist = py27, py35, py36, py37, py38, py39, py310, py311, pypy2, pypy3, mypy skip_missing_interpreters=true + [testenv] deps = pytest commands = pytest + +[testenv:mypy] +deps = mypy +commands = mypy umsgpack From ff5ca534043a6b2d8f5d29b9f3ab9a66a4fd85c4 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Thu, 18 May 2023 03:55:23 -0500 Subject: [PATCH 106/109] add github actions workflow --- .github/workflows/tests.yml | 27 +++++++++++++++++++++++++++ tox.ini | 13 +++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8e0c942 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,27 @@ +name: Tests + +on: + - push + - pull_request + +jobs: + build: + + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy2.7", "pypy3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox tox-gh-actions + - name: Test with tox + run: tox diff --git a/tox.ini b/tox.ini index 6b6086b..7047fd8 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,19 @@ envlist = py27, py35, py36, py37, py38, py39, py310, py311, pypy2, pypy3, mypy skip_missing_interpreters=true +[gh-actions] +python = + 2.7: py27 + 3.5: py35 + 3.6: py36, mypy + 3.7: py37, mypy + 3.8: py38, mypy + 3.9: py39, mypy + 3.10: py310, mypy + 3.11: py311, mypy + pypy-2: pypy2 + pypy-3: pypy3 + [testenv] deps = pytest commands = pytest From 9af0cf7503e97517090d6d2dc38a73600bc6717f Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Thu, 18 May 2023 04:22:58 -0500 Subject: [PATCH 107/109] remove travis config --- .travis.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5c0a602..0000000 --- a/.travis.yml +++ /dev/null @@ -1,28 +0,0 @@ -sudo: false -dist: focal -language: python -install: pip install tox -script: tox - -matrix: - include: - - python: 2.7 - env: TOXENV=py27 - - python: 3.5 - env: TOXENV=py35 - - python: 3.6 - env: TOXENV=py36 - - python: 3.7 - env: TOXENV=py37 - - python: 3.8 - env: TOXENV=py38 - - python: 3.9 - env: TOXENV=py39 - - python: 3.10 - env: TOXENV=py310 - - python: 3.11 - env: TOXENV=py311 - - python: pypy2.7-7.3.1 - env: TOXENV=pypy2 - - python: pypy3.6-7.3.1 - env: TOXENV=pypy3 From 6a2b4a3e4c6f173ab6ed29a679d726f2c2355419 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Thu, 18 May 2023 03:58:04 -0500 Subject: [PATCH 108/109] update tests status badge in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4d40466..485a60b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![Docs Status](https://readthedocs.org/projects/u-msgpack-python/badge/)](https://u-msgpack-python.readthedocs.io/en/latest/) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) +# u-msgpack-python [![Tests Status](https://github.com/vsergeev/u-msgpack-python/actions/workflows/tests.yml/badge.svg)](https://github.com/vsergeev/u-msgpack-python/actions/workflows/tests.yml) [![Docs Status](https://readthedocs.org/projects/u-msgpack-python/badge/)](https://u-msgpack-python.readthedocs.io/en/latest/) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. From 6a9aa1cbbbc828120ed041fd8f467f1efb8dee31 Mon Sep 17 00:00:00 2001 From: "Vanya A. Sergeev" Date: Tue, 16 May 2023 02:09:07 -0500 Subject: [PATCH 109/109] update version and changelog to v2.8.0 --- CHANGELOG.md | 6 ++++++ docs/conf.py | 2 +- setup.py | 2 +- umsgpack/__init__.py | 8 ++++---- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1d433..3cb131b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +* Version 2.8.0 - 05/18/2023 + * Migrate module to package. + * Migrate documentation to Sphinx. + * Improve types and formatting in docstrings. + * Add type stubs. + * Version 2.7.2 - 11/07/2022 * Fix hex formatting of data bytes in Ext string representation. * Contributors diff --git a/docs/conf.py b/docs/conf.py index 6856870..6958620 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,7 +21,7 @@ project = 'u-msgpack-python' copyright = '2013-2023, Vanya A. Sergeev' author = 'Vanya A. Sergeev' -release = '2.7.2' +release = '2.8.0' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/setup.py b/setup.py index 5d5ecc3..1cf6809 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setup( name='u-msgpack-python', - version='2.7.2', + version='2.8.0', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', author_email='v@sergeev.io', diff --git a/umsgpack/__init__.py b/umsgpack/__init__.py index 03f39a1..e7e194f 100644 --- a/umsgpack/__init__.py +++ b/umsgpack/__init__.py @@ -1,4 +1,4 @@ -# u-msgpack-python v2.7.2 - v at sergeev.io +# u-msgpack-python v2.8.0 - v at sergeev.io # https://github.com/vsergeev/u-msgpack-python # # u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -31,7 +31,7 @@ # THE SOFTWARE. # """ -u-msgpack-python v2.7.2 - v at sergeev.io +u-msgpack-python v2.8.0 - v at sergeev.io https://github.com/vsergeev/u-msgpack-python u-msgpack-python is a lightweight MessagePack serializer and deserializer @@ -54,10 +54,10 @@ else: from collections import Hashable -__version__ = "2.7.2" +__version__ = "2.8.0" "Module version string" -version = (2, 7, 2) +version = (2, 8, 0) "Module version tuple"