diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..a6578a3 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +extend-ignore = E501 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..8e0c942 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,27 @@ +name: Tests + +on: + - push + - pull_request + +jobs: + build: + + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy2.7", "pypy3.9"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install tox tox-gh-actions + - name: Test with tox + run: tox diff --git a/.gitignore b/.gitignore index 7d481b4..200d697 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ MANIFEST +build/ dist/ +*.egg-info/ *.pyc *.swp +.tox/ +.cache/ +.pytest_cache/ +docs/_build diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..f334b70 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,13 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + +python: + install: + - requirements: docs/requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cb671a..3cb131b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,79 @@ +* Version 2.8.0 - 05/18/2023 + * Migrate module to package. + * Migrate documentation to Sphinx. + * Improve types and formatting in docstrings. + * Add type stubs. + +* Version 2.7.2 - 11/07/2022 + * Fix hex formatting of data bytes in Ext string representation. + * Contributors + * Stefan Ring, @Ringdingcoder - 309b1a9 + +* Version 2.7.1 - 10/24/2020 + * Add Ext type value validation to Ext class and `ext_serializable()` decorator. + * Change string formatting from `%` to `.format()` throughout codebase. + +* Version 2.7.0 - 08/01/2020 + * Add support for packing subclasses of `ext_serializable()` application classes. + * Contributors + * Gabe Appleton, @gappleto97 - 620f6ef + +* Version 2.6.0 - 04/25/2020 + * Add `use_tuple` option to unpacking functions for unpacking MessagePack arrays into tuples. + * Add `ext_serializable()` decorator for registration of application classes with Ext types for automatic packing and unpacking. + * Contributors + * Gabe Appleton, @gappleto97 - original idea behind 8aa0ee8 in https://github.com/vsergeev/u-msgpack-python/pull/37 + +* Version 2.5.2 - 08/15/2019 + * Fix DeprecationWarning about using ABCs from 'collections' on Python 3.7. + * Contributors + * Gabe Appleton, @gappleto97 - 5ece62a + +* Version 2.5.1 - 03/03/2019 + * Fix handling of naive/aware datetime objects when packing the timestamp extension type. + * Add handling for short reads during file object unpacking. + * Make Ext base class a new-style object for cleaner inheritance in Python 2. + * Improve length comparisons and instance checks for minor performance improvement. + * Contributors + * Gabe Appleton, @gappleto97 - 28907ba, 76751f3, 7a9b717 + * DisposaBoy, @DisposaBoy - 50b1dd3 + +* Version 2.5.0 - 03/31/2018 + * Add support for the timestamp extension type. + * Fix tests on big endian platforms + * Contributors + * Sergei Trofimovich, @trofi - 16510e9 + +* Version 2.4.1 - 04/25/2017 + * Fix module version tuple inconsistency. + +* Version 2.4.0 - 04/20/2017 + * Add hash special method to Ext class. + * Add packing option to force floating point precision. + * Make codebase PEP 8 compliant. + * Add support for tox automated testing and use it in CI. + * Contributors + * Fabien Fleutot, @fab13n - 4c461ed, bdeee20 + * Yuhang (Steven) Wang, @yuhangwang - 5f53bcf + * Pedro Rodrigues, @medecau - 2f7b667, 456032c, e60bc5e + +* Version 2.3.0 - 10/19/2016 + * Add `ext_handlers` option to packing and unpacking functions to support application-defined Ext packing and unpacking hooks. + * Add `allow_invalid_utf8` option to unpacking functions to allow unpacking of invalid UTF-8 strings. + * Add hexadecimal prefix to data bytes in string representation of Ext objects. + * Change version number to semantic versioning. + +* Version 2.2 - 09/25/2016 + * Add `use_ordered_dict` option to unpacking functions for unpacking MessagePack maps into the `collections.OrderedDict` type. + * Add support for `bytearray` type to `unpackb`/`loads` functions. + * Fix intermittent unit test failures due to non-deterministic packing of dict test vectors. + * Fix several docstring examples and typos. + * Add license and unit test to source distribution packaging. + * Contributors + * Fairiz 'Fi' Azizi - df59af6 + * INADA Naoki - 23dbf70 + * Jack O'Connor - 7aa0d19 + * Version 2.1 - 05/09/2015 * Improve array and map unpacking performance under Python 2. * Add module `__version__` attribute. @@ -29,4 +105,5 @@ * Version 1.0 - 09/29/2013 * Initial release. - + * Contributors + * Eugene Ma - 496aaa5 diff --git a/LICENSE b/LICENSE index a0fbb2b..79056ac 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ - Copyright (c) 2013-2014 Ivan A. Sergeev + Copyright (c) 2013-2023 vsergeev / Ivan (Vanya) A. Sergeev Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..6ffd061 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE +include test_umsgpack.py diff --git a/README.md b/README.md index 4c7df1c..485a60b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# u-msgpack-python v2.1 +# u-msgpack-python [![Tests Status](https://github.com/vsergeev/u-msgpack-python/actions/workflows/tests.yml/badge.svg)](https://github.com/vsergeev/u-msgpack-python/actions/workflows/tests.yml) [![Docs Status](https://readthedocs.org/projects/u-msgpack-python/badge/)](https://u-msgpack-python.readthedocs.io/en/latest/) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) -u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, and application-defined ext types. +u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. -u-msgpack-python is currently distributed on PyPI: https://pypi.python.org/pypi/u-msgpack-python and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) +u-msgpack-python is currently distributed as a package on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file module. ## Installation @@ -16,9 +16,9 @@ With easy_install: $ easy_install u-msgpack-python ``` -or simply drop [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) into your project! +or simply drop `umsgpack.py` into your project! ``` text -$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py +$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack/__init__.py -O umsgpack.py ``` ## Examples @@ -27,7 +27,7 @@ Basic Example: ``` python >>> import umsgpack >>> umsgpack.packb({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.unpackb(_) {u'compact': True, u'schema': 0} >>> @@ -35,13 +35,13 @@ Basic Example: A more complicated example: ``` python ->>> umsgpack.packb( [1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", - u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345] ) -'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01' -'\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb' -'@\x00\xfc\xd3Z\x85\x87\x94' +>>> umsgpack.packb([1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", \ +... u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345]) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' >>> umsgpack.unpackb(_) -[1, True, False, 4294967295, {u'foo': '\x80\x01\x02', +[1, True, False, 4294967295, {u'foo': b'\x80\x01\x02', \ u'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] >>> ``` @@ -62,28 +62,61 @@ Streaming serialization with file-like objects: >>> ``` -Encoding and decoding an application-defined ext type: +Serializing and deserializing a raw Ext type: ``` python -# Create an Ext object with type 0x05 and data b"\x01\x02\x03" ->>> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") ->>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) -b'\x82\xadspecial stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> # Create an Ext object with type 5 and data b"\x01\x02\x03" +... foo = umsgpack.Ext(5, b"\x01\x02\x03") +>>> umsgpack.packb({u"stuff": foo, u"awesome": True}) +b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> >>> bar = umsgpack.unpackb(_) ->>> print(bar["special stuff"]) -Ext Object (Type: 0x05, Data: 01 02 03) ->>> bar["special stuff"].type +>>> print(bar['stuff']) +Ext Object (Type: 5, Data: 0x01 0x02 0x03) +>>> bar['stuff'].type 5 ->>> bar["special stuff"].data +>>> bar['stuff'].data b'\x01\x02\x03' >>> ``` -Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: +Serializing and deserializing application-defined types with `ext_serializable()`: +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... def packb(self): +... return struct.pack(">ii", self.x, self.y) +... @staticmethod +... def unpackb(data): +... return Point(*struct.unpack(">ii", data)) +... +>>> umsgpack.packb(Point(1, 2)) +b'\xd7P\x00\x00\x00\x01\x00\x00\x00\x02' +>>> umsgpack.unpackb(_) +Point(x=1, y=2) +>>> +``` +Serializing and deserializing application-defined types with Ext handlers: +``` python +>>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], +... ext_handlers = { +... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xd6@0.31' +>>> umsgpack.unpackb(_, +... ext_handlers = { +... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), +... }) +[(1+2j), Decimal('0.31')] +>>> +``` + +Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: ``` python ->>> import umsgpack >>> umsgpack.dumps({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.loads(_) {u'compact': True, u'schema': 0} >>> @@ -97,171 +130,24 @@ Python standard library style names `dump`, `dumps`, `load`, `loads` are also av >>> ``` -## Streaming Serialization and Deserialization - -The streaming `pack()`/`dump()` and `unpack()`/`load()` functions allow packing and unpacking objects directly to and from a stream, respectively. Streaming may be necessary when unpacking serialized bytes whose size is unknown in advance, or it may be more convenient and efficient when working directly with stream objects (e.g. files or stream sockets). - -`pack(obj, fp)` / `dump(obj, fp)` serialize Python object `obj` to a `.write()` supporting file-like object `fp`. +## Documentation -``` python ->>> class Foo: -... def write(self, data): -... # write 'data' bytes -... pass -... ->>> f = Foo() ->>> umsgpack.pack({u"compact": True, u"schema": 0}, f) ->>> -``` +Documentation is hosted at [https://u-msgpack-python.readthedocs.io](https://u-msgpack-python.readthedocs.io). -`unpack(fp)` / `load(fp)` deserialize a Python object from a `.read()` supporting file-like object `fp`. +To build documentation locally with Sphinx, run: -``` python ->>> class Bar: -... def read(self, n): -... # read and return 'n' number of bytes -... return b"\x01"*n -... ->>> f = Bar() ->>> umsgpack.unpack(f) -1 ->>> ``` - -## Compatibility Mode - -u-msgpack-python offers a compatibility mode for the [old specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md) to handle the old "raw" bytes msgpack type. When the compatibility mode is enabled, u-msgpack-python will serialize both unicode strings and bytes into the old "raw" msgpack type, and deserialize the "raw" msgpack type into bytes. To enable compatibility mode, simply set the `compatibility` boolean of the umsgpack module to `True`. - -``` python ->>> umsgpack.compatibility = True ->>> ->>> umsgpack.packb([u"some string", b"some bytes"]) -b'\x92\xabsome string\xaasome bytes' ->>> umsgpack.unpackb(_) -[b'some string', b'some bytes'] ->>> +cd docs +make html ``` -## Exceptions +Sphinx will produce the HTML documentation in `docs/_build/html/`. -### Packing Exceptions - -If an error occurs during packing, umsgpack will raise an exception derived from `umsgpack.PackException`. All possible packing exceptions are described below. - -* `UnsupportedTypeException`: Object type not supported for packing. - - ``` python - >>> # Attempt to pack set type - ... umsgpack.packb(set([1,2,3])) - ... - umsgpack.UnsupportedTypeException: unsupported type: - >>> - - >>> # Attempt to pack > 64-bit unsigned int - ... umsgpack.packb(2**64) - ... - umsgpack.UnsupportedTypeException: huge unsigned int - >>> - ``` - -### Unpacking Exceptions - -If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise a `TypeError` exception. If an error occurs during unpacking, umsgpack will raise an exception derived from `umsgpack.UnpackException`. All possible unpacking exceptions are described below. - -* `TypeError`: Packed data is not type `str` (Python 2), or not type `bytes` (Python 3). - - ``` python - # Attempt to unpack non-str type data in Python 2 - >>> umsgpack.unpackb(u"no good") - ... - TypeError: expected packed data as type 'str' - >>> - - # Attempt to unpack non-bytes type data in Python 3 - >>> umsgpack.unpackb("no good") - ... - TypeError: expected packed data as type 'bytes' - >>> - ``` - -* `InsufficientDataException`: Insufficient data to unpack the encoded object. - - ``` python - # Attempt to unpack a cut-off encoded 32-bit unsigned int - >>> umsgpack.unpackb(b"\xce\xff\xff\xff") - ... - umsgpack.InsufficientDataException - >>> - - # Attempt to unpack an array of length 2 missing the second item - >>> umsgpack.unpackb(b"\x92\xc2") - ... - umsgpack.InsufficientDataException - >>> - - ``` -* `InvalidStringException`: Invalid UTF-8 string encountered during unpacking. - - String bytes are strictly decoded with UTF-8. This exception is thrown if UTF-8 decoding of string bytes fails. - - ``` python - # Attempt to unpack the string b"\x80\x81" - >>> umsgpack.unpackb(b"\xa2\x80\x81") - ... - umsgpack.InvalidStringException: unpacked string is not utf-8 - >>> - ``` - -* `ReservedCodeException`: Reserved code encountered during unpacking. - - ``` python - # Attempt to unpack reserved code 0xc1 - >>> umsgpack.unpackb(b"\xc1") - ... - umsgpack.ReservedCodeException: reserved code encountered: 0xc1 - >>> - ``` - -* `UnhashableKeyException`: Unhashable key encountered during map unpacking. The packed map cannot be unpacked into a Python dictionary. - - Python dictionaries only support keys that are instances of `collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a msgpack encoding, it cannot be unpacked into a valid Python dictionary. - - ``` python - # Attempt to unpack { {} : False } - >>> umsgpack.unpackb(b"\x82\x80\xc2") - ... - umsgpack.UnhashableKeyException: encountered unhashable key type: {}, - >>> - ``` - -* `DuplicateKeyException`: Duplicate key encountered during map unpacking. - - Python dictionaries do not support duplicate keys, but msgpack maps may be encoded with duplicate keys. - - ``` python - # Attempt to unpack { 1: True, 1: False } - >>> umsgpack.unpackb(b"\x82\x01\xc3\x01\xc2") - ... - umsgpack.DuplicateKeyException: encountered duplicate key: 1, - >>> - ``` - -## Behavior Notes - -* Python 2 - * `unicode` type objects are packed into, and unpacked from, the msgpack `string` format - * `str` type objects are packed into, and unpacked from, the msgpack `binary` format -* Python 3 - * `str` type objects are packed into, and unpacked from, the msgpack `string` format - * `bytes` type objects are packed into, and unpacked from, the msgpack `binary` format -* The msgpack string format is strictly decoded with UTF-8 -- an exception is thrown if the string bytes cannot be decoded into a valid UTF-8 string -* The msgpack array format is unpacked into a Python list, unless it is the key of a map, in which case it is unpacked into a Python tuple -* Python tuples and lists are both packed into the msgpack array format -* Python float types are packed into the msgpack float32 or float64 format depending on the system's `sys.float_info` +Run `make help` to see other output targets (LaTeX, man, text, etc.). ## Testing -u-msgpack-python's included unit tests may be run with `test_umsgpack.py`, under your favorite interpreter. +The included unit tests may be run with `test_umsgpack.py`, under your favorite interpreter. ``` text $ python2 test_umsgpack.py @@ -270,7 +156,13 @@ $ pypy test_umsgpack.py $ pypy3 test_umsgpack.py ``` +Alternatively, you can use `tox` or `detox` to test multiple Python versions at once. + +``` text +$ pip install tox +$ tox +``` + ## License u-msgpack-python is MIT licensed. See the included `LICENSE` file for more details. - diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d4bb2cb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..d53f6d4 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,105 @@ +# API + +## Packing + +```{eval-rst} +.. autofunction:: umsgpack.packb + +Also available under the ``umsgpack.dumps()`` alias. +``` + +```{eval-rst} +.. autofunction:: umsgpack.pack + +Also available under the ``umsgpack.dump()`` alias. +``` + +## Unpacking + +```{eval-rst} +.. autofunction:: umsgpack.unpackb + +Also available under the ``umsgpack.loads()`` alias. +``` + +```{eval-rst} +.. autofunction:: umsgpack.unpack + +Also available under the ``umsgpack.load()`` alias. +``` + +## Packing Exceptions + +```{eval-rst} +.. autoexception:: umsgpack.PackException +``` + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTypeException +``` + +## Unpacking Exceptions + +```{eval-rst} +.. autoexception:: umsgpack.UnpackException +``` + +```{eval-rst} +.. autoexception:: umsgpack.InsufficientDataException +``` + +```{eval-rst} +.. autoexception:: umsgpack.InvalidStringException +``` + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTimestampException +``` + +```{eval-rst} +.. autoexception:: umsgpack.ReservedCodeException +``` + +```{eval-rst} +.. autoexception:: umsgpack.UnhashableKeyException +``` + +```{eval-rst} +.. autoexception:: umsgpack.DuplicateKeyException +``` + +## Ext Class + +```{eval-rst} +.. autoclass:: umsgpack.Ext + :member-order: bysource + :special-members: __init__, __eq__, __ne__, __str__, __hash__ +``` + +## Ext Serializable Decorator + +```{eval-rst} +.. autodecorator:: umsgpack.ext_serializable +``` + +## Invalid String Class + +```{eval-rst} +.. autoclass:: umsgpack.InvalidString +``` + +## Attributes + +```{eval-rst} +.. autodata:: umsgpack.compatibility +``` + +## Constants + +```{eval-rst} +.. autodata:: umsgpack.version +``` + +```{eval-rst} +.. autodata:: umsgpack.__version__ +``` diff --git a/docs/behavior-notes.md b/docs/behavior-notes.md new file mode 100644 index 0000000..25474ee --- /dev/null +++ b/docs/behavior-notes.md @@ -0,0 +1,33 @@ +# Behavior Notes + +* Python 2 + * `unicode` type objects are packed into, and unpacked from, the MessagePack + `string` format + * `str` type objects are packed into, and unpacked from, the MessagePack + `binary` format +* Python 3 + * `str` type objects are packed into, and unpacked from, the MessagePack + `string` format + * `bytes` type objects are packed into, and unpacked from, the MessagePack + `binary` format +* The MessagePack string format is strictly decoded with UTF-8 — an exception + is thrown if the string bytes cannot be decoded into a valid UTF-8 string, + unless the `allow_invalid_utf8` option is enabled +* The MessagePack array format is unpacked into a Python list, unless it is the + key of a map, in which case it is unpacked into a Python tuple +* Python tuples and lists are both packed into the MessagePack array format +* Python float types are packed into the MessagePack float32 or float64 format + depending on the system's `sys.float_info` +* The Python `datetime.datetime` type is packed into, and unpacked from, the + MessagePack `timestamp` format + * Note that the Python `datetime.datetime` type only supports microsecond + resolution, while the MessagePack `timestamp` format supports nanosecond + resolution. Timestamps with finer than microsecond resolution will lose + precision during unpacking. Users may override the packing and unpacking + of the MessagePack `timestamp` format with a custom type for alternate + behavior. + * Both naive and aware timestamp are supported. Naive timestamps are packed + as if they are in the UTC timezone. Timestamps are always unpacked as + aware `datetime.datetime` objects in the UTC timezone. +* Ext type handlers specified in the optional `ext_handlers` dictionary will + override `ext_serializable()` classes during packing and unpacking diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..6958620 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,47 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. + +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + +import umsgpack + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'u-msgpack-python' +copyright = '2013-2023, Vanya A. Sergeev' +author = 'Vanya A. Sergeev' +release = '2.8.0' + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.viewcode', + 'sphinx_rtd_theme', + 'myst_parser', +] + +templates_path = ['_templates'] +exclude_patterns = ['_build'] + +pygments_style = 'sphinx' + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = 'sphinx_rtd_theme' +html_static_path = ['_static'] +html_show_sourcelink = False diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..6d63f8c --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,117 @@ +# Examples + +Basic Example: + +``` python +>>> import umsgpack +>>> umsgpack.packb({u"compact": True, u"schema": 0}) +b'\x82\xa7compact\xc3\xa6schema\x00' +>>> umsgpack.unpackb(_) +{u'compact': True, u'schema': 0} +>>> +``` + +A more complicated example: + +``` python +>>> umsgpack.packb([1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", \ +... u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345]) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' +>>> umsgpack.unpackb(_) +[1, True, False, 4294967295, {u'foo': b'\x80\x01\x02', \ + u'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] +>>> +``` + +Streaming serialization with file-like objects: + +``` python +>>> f = open('test.bin', 'wb') +>>> umsgpack.pack({u"compact": True, u"schema": 0}, f) +>>> umsgpack.pack([1,2,3], f) +>>> f.close() +>>> +>>> f = open('test.bin', 'rb') +>>> umsgpack.unpack(f) +{u'compact': True, u'schema': 0} +>>> umsgpack.unpack(f) +[1, 2, 3] +>>> f.close() +>>> +``` + +Serializing and deserializing a raw Ext type: + +``` python +>>> # Create an Ext object with type 5 and data b"\x01\x02\x03" +... foo = umsgpack.Ext(5, b"\x01\x02\x03") +>>> umsgpack.packb({u"stuff": foo, u"awesome": True}) +b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> +>>> bar = umsgpack.unpackb(_) +>>> print(bar['stuff']) +Ext Object (Type: 5, Data: 0x01 0x02 0x03) +>>> bar['stuff'].type +5 +>>> bar['stuff'].data +b'\x01\x02\x03' +>>> +``` + +Serializing and deserializing application-defined types with + +`ext_serializable()`: +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... def packb(self): +... return struct.pack(">ii", self.x, self.y) +... @staticmethod +... def unpackb(data): +... return Point(*struct.unpack(">ii", data)) +... +>>> umsgpack.packb(Point(1, 2)) +b'\xd7P\x00\x00\x00\x01\x00\x00\x00\x02' +>>> umsgpack.unpackb(_) +Point(x=1, y=2) +>>> +``` + +Serializing and deserializing application-defined types with Ext handlers: + +``` python +>>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], +... ext_handlers = { +... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xd6@0.31' +>>> umsgpack.unpackb(_, +... ext_handlers = { +... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), +... }) +[(1+2j), Decimal('0.31')] +>>> +``` + +Python standard library style names `dump`, `dumps`, `load`, `loads` are also +available: + +``` python +>>> umsgpack.dumps({u"compact": True, u"schema": 0}) +b'\x82\xa7compact\xc3\xa6schema\x00' +>>> umsgpack.loads(_) +{u'compact': True, u'schema': 0} +>>> +>>> f = open('test.bin', 'wb') +>>> umsgpack.dump({u"compact": True, u"schema": 0}, f) +>>> f.close() +>>> +>>> f = open('test.bin', 'rb') +>>> umsgpack.load(f) +{u'compact': True, u'schema': 0} +>>> +``` diff --git a/docs/extension.md b/docs/extension.md new file mode 100644 index 0000000..c967809 --- /dev/null +++ b/docs/extension.md @@ -0,0 +1,111 @@ +# Extension Types + +u-msgpack-python supports two mechanisms for packing and unpacking MessagePack +Ext types: the `ext_handlers` keyword option, and the `ext_serializable()` +decorator. + +## Ext Handlers + +The packing functions accept an optional `ext_handlers` dictionary that maps +custom types to callables that pack the type into an Ext object. The callable +should accept the custom type object as an argument and return a packed +`umsgpack.Ext` object. + +Example for packing `set`, `complex`, and `decimal.Decimal` types into Ext +objects with type codes 0x20, 0x30, and 0x40, respectively: + +``` python +>>> umsgpack.packb([1, True, {"foo", 2}, complex(3, 4), decimal.Decimal("0.31")], +... ext_handlers = { +... set: lambda obj: umsgpack.Ext(0x20, umsgpack.packb(list(obj))), +... complex: lambda obj: umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... decimal.Decimal: lambda obj: umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31' +>>> +``` +Similarly, the unpacking functions accept an optional `ext_handlers` dictionary +that maps Ext type codes to callables that unpack the Ext into a custom object. +The callable should accept a `umsgpack.Ext` object as an argument and return an +unpacked custom type object. + +Example for unpacking Ext objects with type codes 0x20, 0x30, and 0x40, into +`set`, `complex`, and `decimal.Decimal` typed objects, respectively: + +``` python +>>> umsgpack.unpackb(b'\x95\x01\xc3\xc7\x06 \x92\xa3foo\x02\xd70\x00\x00@@\x00\x00\x80@\xd6@0.31', +... ext_handlers = { +... 0x20: lambda ext: set(umsgpack.unpackb(ext.data)), +... 0x30: lambda ext: complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: decimal.Decimal(ext.data.decode()), +... }) +[1, True, {'foo', 2}, (3+4j), Decimal('0.31')] +>>> +``` + +Example for packing and unpacking a custom class: + +``` python +class Point(object): + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z + + def __str__(self): + return "Point({}, {}, {})".format(self.x, self.y, self.z) + + def pack(self): + return struct.pack(">iii", self.x, self.y, self.z) + + @staticmethod + def unpack(data): + return Point(*struct.unpack(">iii", data)) + +# Pack +obj = Point(1,2,3) +data = umsgpack.packb(obj, ext_handlers = {Point: lambda obj: umsgpack.Ext(0x10, obj.pack())}) + +# Unpack +obj = umsgpack.unpackb(data, ext_handlers = {0x10: lambda ext: Point.unpack(ext.data)}) +print(obj) # -> Point(1, 2, 3) +``` + +## Ext Serializable + +The `ext_serializable()` decorator registers application classes for automatic +packing and unpacking with the specified Ext type. The decorator accepts the +Ext type code as an argument. The application class should implement a +`packb()` method that returns serialized bytes, and an `unpackb()` class method +or static method that accepts serialized bytes and returns an instance of the +application class. + +Example for registering, packing, and unpacking a custom class with Ext type +code 0x10: + +``` python +@umsgpack.ext_serializable(0x10) +class Point(object): + def __init__(self, x, y, z): + self.x = x + self.y = y + self.z = z + + def __str__(self): + return "Point({}, {}, {})".format(self.x, self.y, self.z) + + def packb(self): + return struct.pack(">iii", self.x, self.y, self.z) + + @staticmethod + def unpackb(data): + return Point(*struct.unpack(">iii", data)) + +# Pack +obj = Point(1,2,3) +data = umsgpack.packb(obj) + +# Unpack +obj = umsgpack.unpackb(data) +print(obj) # -> Point(1, 2, 3) +``` diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..f18f8ff --- /dev/null +++ b/docs/index.md @@ -0,0 +1,28 @@ +# Welcome to u-msgpack-python's documentation! + +[u-msgpack-python](https://github.com/vsergeev/u-msgpack-python) is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). In particular, it supports the new binary, UTF-8 string, application-defined ext, and timestamp types. + +u-msgpack-python is currently distributed as a package on [PyPI](https://pypi.python.org/pypi/u-msgpack-python) and as a single file module. + +## Contents + + +```{toctree} +:maxdepth: 1 + +Home +installation.md +examples.md +packing.md +unpacking.md +streaming.md +extension.md +api.md +behavior-notes.md +license.md +``` + +## Indices and tables + +* [](genindex) +* [](search) diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 0000000..8651fd6 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,19 @@ +# Installation + +Install with pip: + +``` text +$ pip install u-msgpack-python +``` + +Install with easy_install: + +``` text +$ easy_install u-msgpack-python +``` + +or simply drop `umsgpack.py` into your project! + +``` text +$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack/__init__.py -O umsgpack.py +``` diff --git a/docs/license.md b/docs/license.md new file mode 100644 index 0000000..9594f3e --- /dev/null +++ b/docs/license.md @@ -0,0 +1,3 @@ +# License + +u-msgpack-python is MIT licensed. See the included [`LICENSE`](https://raw.github.com/vsergeev/u-msgpack-python/master/LICENSE) file for more details. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..32bb245 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/packing.md b/docs/packing.md new file mode 100644 index 0000000..b5293db --- /dev/null +++ b/docs/packing.md @@ -0,0 +1,103 @@ +# Packing + +## Example + +``` python +>>> umsgpack.packb([1, True, False, 0xffffffff, {'foo': b'\x80\x01\x02', \ +... 'bar': [1,2,3, {'a': [1,2,3,{}]}]}, -1, 2.12345]) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' +>>> +``` + +## API + +```{eval-rst} +.. autofunction:: umsgpack.packb + :noindex: + +Also available under the ``umsgpack.dumps()`` alias. +``` + +## Options + +### Ext Handlers + +See the [Extension Types](extension.md) section. + +### Float Precision + +The packing functions provide a `force_float_precision` option to force packing +of floats into the specified precision: `"single"` for IEEE-754 +single-precision floats, or `"double"` for IEEE-754 double-precision floats. + +``` python +>>> # Force float packing to single-precision floats +... umsgpack.packb(2.5, force_float_precision="single") +b'\xca@ \x00\x00' +>>> # Force float packing to double-precision floats +... umsgpack.packb(2.5, force_float_precision="double") +b'\xcb@\x04\x00\x00\x00\x00\x00\x00' +>>> +``` + +### Old Specification Compatibility Mode + +The compatibility mode supports the "raw" bytes MessagePack type from the [old +specification](https://github.com/msgpack/msgpack/blob/master/spec-old.md). +When the module-wide `compatibility` attribute is enabled, both unicode strings +and bytes will be serialized into the "raw" MessagePack type, and the "raw" +MessagePack type will be deserialized into bytes. + +``` python +>>> umsgpack.compatibility = True +>>> +>>> umsgpack.packb([u"some string", b"some bytes"]) +b'\x92\xabsome string\xaasome bytes' +>>> umsgpack.unpackb(_) +[b'some string', b'some bytes'] +>>> +``` + +## Exceptions + +If an error occurs during packing, u-msgpack-python will raise an exception +derived from `umsgpack.PackException`. Possible packing exceptions are +described below. + +### UnsupportedTypeException + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTypeException + :noindex: +``` + +``` python +>>> # Attempt to pack set type +... umsgpack.packb(set([1,2,3])) +... +umsgpack.UnsupportedTypeException: unsupported type: +>>> + +>>> # Attempt to pack > 64-bit unsigned int +... umsgpack.packb(2**64) +... +umsgpack.UnsupportedTypeException: huge unsigned int +>>> +``` + +### NotImplementedError + +Ext serializable class is missing implementation of `packb()`. + +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... pass +... +>>> umsgpack.packb(Point(1, 2)) +... +NotImplementedError: Ext serializable class is missing implementation of packb() +>>> +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..7c98d98 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx-rtd-theme>=1.2.0 +myst-parser diff --git a/docs/streaming.md b/docs/streaming.md new file mode 100644 index 0000000..2e30fdc --- /dev/null +++ b/docs/streaming.md @@ -0,0 +1,54 @@ +# Streaming + +The streaming `pack()` and `unpack()` functions allow packing and unpacking +objects directly to and from a stream, respectively. Streaming may be necessary +when unpacking serialized bytes whose size is unknown in advance, or it may be +more convenient and efficient when working directly with stream objects (e.g. +files or stream sockets). + +## Packing + +`pack(obj, fp)` serializes Python object `obj` to a `.write()` supporting +file-like object `fp`. + +``` python +>>> class Foo: +... def write(self, data): +... # write 'data' bytes +... pass +... +>>> f = Foo() +>>> umsgpack.pack({u"compact": True, u"schema": 0}, f) +>>> +``` + +```{eval-rst} +.. autofunction:: umsgpack.pack + :noindex: + +Also available under the ``umsgpack.dump()`` alias. +``` + +## Unpacking + +`unpack(fp)` deserializes a Python object from a `.read()` supporting file-like +object `fp`. + +``` python +>>> class Bar: +... def read(self, n): +... # read and return 'n' number of bytes +... return b"\x01"*n +... +>>> f = Bar() +>>> umsgpack.unpack(f) +1 +>>> +``` + +```{eval-rst} +.. autofunction:: umsgpack.unpack + :noindex: + +Also available under the ``umsgpack.load()`` alias. +``` diff --git a/docs/unpacking.md b/docs/unpacking.md new file mode 100644 index 0000000..43ccf94 --- /dev/null +++ b/docs/unpacking.md @@ -0,0 +1,225 @@ +# Unpacking + +## Example + +``` python +>>> umsgpack.unpackb(b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3\ +foo\xc4\x03\x80\x01\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\ +\x03\x80\xff\xcb@\x00\xfc\xd3Z\x85\x87\x94') +[1, True, False, 4294967295, {'foo': b'\x80\x01\x02', \ + 'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] +>>> +``` + +## API + +```{eval-rst} +.. autofunction:: umsgpack.unpackb + :noindex: + +Also available under the ``umsgpack.loads()`` alias. +``` + +## Options + +### Ext Handlers + +See the [Extension Types](extension.md) section. + +### Ordered Dictionaries + +The unpacking functions provide a `use_ordered_dict` option to unpack +MessagePack maps into the `collections.OrderedDict` type, rather than the +unordered `dict` type, to preserve the order of deserialized MessagePack maps. +Note that as of Python 3.6, dictionaries are insertion ordered by default. + +``` python +>>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') +{'compact': True, 'schema': 0} +>>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00', use_ordered_dict=True) +OrderedDict([('compact', True), ('schema', 0)]) +>>> +``` + +### Tuples + +The unpacking functions provide a `use_tuple` option to unpack MessagePack +arrays into tuples, rather than lists. + +``` python +>>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03') +['a', True, [1, [2, 3]]] +>>> umsgpack.unpackb(b'\x93\xa1a\xc3\x92\x01\x92\x02\x03', use_tuple=True) +('a', True, (1, (2, 3))) +>>> +``` + +### Invalid UTF-8 Strings + +The unpacking functions provide an `allow_invalid_utf8` option to unpack +MessagePack strings with invalid UTF-8 into the `umsgpack.InvalidString` type, +instead of throwing an exception. The `umsgpack.InvalidString` type is a +subclass of `bytes`, and can be used like any other `bytes` object. + +``` python +>>> # Attempt to unpack invalid UTF-8 string +... umsgpack.unpackb(b'\xa4\x80\x01\x02\x03') +... +umsgpack.InvalidStringException: unpacked string is invalid utf-8 +>>> umsgpack.unpackb(b'\xa4\x80\x01\x02\x03', allow_invalid_utf8=True) +b'\x80\x01\x02\x03' +>>> +``` + +## Exceptions + +If a non-byte-string argument is passed to `umsgpack.unpackb()`, it will raise +a `TypeError` exception. If an error occurs during unpacking, u-msgpack-python +will raise an exception derived from `umsgpack.UnpackException`. Possible +unpacking exceptions are described below. + + +### TypeError + +Packed data is not type `str` (Python 2), or not type `bytes` (Python 3). + +``` python +# Attempt to unpack non-str type data in Python 2 +>>> umsgpack.unpackb(u"no good") +... +TypeError: expected packed data as type 'str' +>>> + +# Attempt to unpack non-bytes type data in Python 3 +>>> umsgpack.unpackb("no good") +... +TypeError: expected packed data as type 'bytes' +>>> +``` + +### InsufficientDataException + +```{eval-rst} +.. autoexception:: umsgpack.InsufficientDataException + :noindex: +``` + +``` python +# Attempt to unpack a cut-off serialized 32-bit unsigned int +>>> umsgpack.unpackb(b"\xce\xff\xff\xff") +... +umsgpack.InsufficientDataException +>>> + +# Attempt to unpack an array of length 2 missing the second item +>>> umsgpack.unpackb(b"\x92\xc2") +... +umsgpack.InsufficientDataException +>>> +``` + +### InvalidStringException + +```{eval-rst} +.. autoexception:: umsgpack.InvalidStringException + :noindex: +``` + +String bytes are strictly decoded with UTF-8. This exception is thrown if UTF-8 +decoding of string bytes fails. Use the `allow_invalid_utf8` option to unpack +invalid MessagePack strings into byte strings. + +``` python +# Attempt to unpack invalid UTF-8 string +>>> umsgpack.unpackb(b"\xa2\x80\x81") +... +umsgpack.InvalidStringException: unpacked string is invalid utf-8 +>>> +``` + +### UnsupportedTimestampException + +```{eval-rst} +.. autoexception:: umsgpack.UnsupportedTimestampException + :noindex: +``` + +The official timestamp extension type supports 32-bit, 64-bit and 96-bit +formats. This exception is thrown if a timestamp extension type with an +unsupported format is encountered. + +``` python +# Attempt to unpack invalid timestamp +>>> umsgpack.unpackb(b"\xd5\xff\x01\x02") +... +umsgpack.UnsupportedTimestampException: unsupported timestamp with data length 2 +>>> +``` + +### ReservedCodeException + +```{eval-rst} +.. autoexception:: umsgpack.ReservedCodeException + :noindex: +``` + +``` python +# Attempt to unpack reserved code 0xc1 +>>> umsgpack.unpackb(b"\xc1") +... +umsgpack.ReservedCodeException: reserved code encountered: 0xc1 +>>> +``` + +### UnhashableKeyException + +```{eval-rst} +.. autoexception:: umsgpack.UnhashableKeyException + :noindex: +``` + +Python dictionaries only support keys that are instances of +`collections.Hashable`, so while the map `{ { u'abc': True } : 5 }` has a +MessagePack serialization, it cannot be unpacked into a valid Python +dictionary. + +``` python +# Attempt to unpack { {} : False } +>>> umsgpack.unpackb(b"\x82\x80\xc2") +... +umsgpack.UnhashableKeyException: encountered unhashable key type: {}, +>>> +``` + +### DuplicateKeyException + +```{eval-rst} +.. autoexception:: umsgpack.DuplicateKeyException + :noindex: +``` + +Python dictionaries do not support duplicate keys, but MessagePack maps may be +serialized with duplicate keys. + +``` python +# Attempt to unpack { 1: True, 1: False } +>>> umsgpack.unpackb(b"\x82\x01\xc3\x01\xc2") +... +umsgpack.DuplicateKeyException: encountered duplicate key: 1, +>>> +``` + +### NotImplementedError + +Ext serializable class is missing implementation of `unpackb()`. + +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... pass +... +>>> umsgpack.unpackb(b'\xd7\x50\x00\x00\x00\x01\x00\x00\x00\x02') +... +NotImplementedError: Ext serializable class is missing implementation of unpackb() +>>> +``` diff --git a/msgpack.org.md b/msgpack.org.md index 7cd2c1b..d508bf0 100644 --- a/msgpack.org.md +++ b/msgpack.org.md @@ -1,8 +1,8 @@ -# u-msgpack-python v2.1 +# u-msgpack-python [![Build Status](https://app.travis-ci.com/vsergeev/u-msgpack-python.svg?branch=master)](https://app.travis-ci.com/github/vsergeev/u-msgpack-python) [![GitHub release](https://img.shields.io/github/release/vsergeev/u-msgpack-python.svg?maxAge=7200)](https://github.com/vsergeev/u-msgpack-python) [![License](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/vsergeev/u-msgpack-python/blob/master/LICENSE) -u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with both Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). +u-msgpack-python is a lightweight [MessagePack](http://msgpack.org/) serializer and deserializer module written in pure Python, compatible with Python 2 and 3, as well CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest [MessagePack specification](https://github.com/msgpack/msgpack/blob/master/spec.md). -u-msgpack-python is currently distributed on PyPI: https://pypi.python.org/pypi/u-msgpack-python and as a single file: [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) +u-msgpack-python is currently distributed as a package on PyPI: https://pypi.python.org/pypi/u-msgpack-python and as a single file module. ## Installation @@ -16,9 +16,9 @@ With easy_install: $ easy_install u-msgpack-python ``` -or simply drop [umsgpack.py](https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py) into your project! +or simply drop `umsgpack.py` into your project! ``` text -$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack.py +$ wget https://raw.github.com/vsergeev/u-msgpack-python/master/umsgpack/__init__.py -O umsgpack.py ``` ## Examples @@ -27,7 +27,7 @@ Basic Example: ``` python >>> import umsgpack >>> umsgpack.packb({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.unpackb(_) {u'compact': True, u'schema': 0} >>> @@ -36,13 +36,13 @@ Basic Example: A more complicated example: ``` python >>> umsgpack.packb( - [1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", - u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345] ) -'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01' -'\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb' -'@\x00\xfc\xd3Z\x85\x87\x94' +... [1, True, False, 0xffffffff, {u"foo": b"\x80\x01\x02", +... u"bar": [1,2,3, {u"a": [1,2,3,{}]}]}, -1, 2.12345] ) +b'\x97\x01\xc3\xc2\xce\xff\xff\xff\xff\x82\xa3foo\xc4\x03\x80\x01\ +\x02\xa3bar\x94\x01\x02\x03\x81\xa1a\x94\x01\x02\x03\x80\xff\xcb\ +@\x00\xfc\xd3Z\x85\x87\x94' >>> umsgpack.unpackb(_) -[1, True, False, 4294967295, {u'foo': '\x80\x01\x02', +[1, True, False, 4294967295, {u'foo': b'\x80\x01\x02', \ u'bar': [1, 2, 3, {u'a': [1, 2, 3, {}]}]}, -1, 2.12345] >>> ``` @@ -63,29 +63,66 @@ Streaming serialization with file-like objects: >>> ``` -Encoding and decoding an application-defined ext type: +Serializing and deserializing a raw Ext type: ``` python ->>> # Create an Ext object with type 0x05 and data b"\x01\x02\x03" -... foo = umsgpack.Ext(0x05, b"\x01\x02\x03") ->>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) -b'\x82\xadspecial stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> # Create an Ext object with type 5 and data b"\x01\x02\x03" +... foo = umsgpack.Ext(5, b"\x01\x02\x03") +>>> umsgpack.packb({u"stuff": foo, u"awesome": True}) +b'\x82\xa5stuff\xc7\x03\x05\x01\x02\x03\xa7awesome\xc3' +>>> >>> bar = umsgpack.unpackb(_) ->>> print(bar["special stuff"]) -Ext Object (Type: 0x05, Data: 01 02 03) ->>> bar["special stuff"].type +>>> print(bar['stuff']) +Ext Object (Type: 5, Data: 0x01 0x02 0x03) +>>> bar['stuff'].type 5 ->>> bar["special stuff"].data +>>> bar['stuff'].data b'\x01\x02\x03' >>> ``` +Serializing and deserializing application-defined types with `ext_serializable()`: +``` python +>>> @umsgpack.ext_serializable(0x50) +... class Point(collections.namedtuple('Point', ['x', 'y'])): +... def packb(self): +... return struct.pack(">ii", self.x, self.y) +... @staticmethod +... def unpackb(data): +... return Point(*struct.unpack(">ii", data)) +... +>>> umsgpack.packb(Point(1, 2)) +b'\xd7P\x00\x00\x00\x01\x00\x00\x00\x02' +>>> umsgpack.unpackb(_) +Point(x=1, y=2) +>>> +``` + +Serializing and deserializing application-defined types with Ext handlers: +``` python +>>> umsgpack.packb([complex(1,2), decimal.Decimal("0.31")], +... ext_handlers = { +... complex: lambda obj: +... umsgpack.Ext(0x30, struct.pack("ff", obj.real, obj.imag)), +... decimal.Decimal: lambda obj: +... umsgpack.Ext(0x40, str(obj).encode()), +... }) +b'\x92\xd70\x00\x00\x80?\x00\x00\x00@\xd6@0.31' +>>> umsgpack.unpackb(_, +... ext_handlers = { +... 0x30: lambda ext: +... complex(*struct.unpack("ff", ext.data)), +... 0x40: lambda ext: +... decimal.Decimal(ext.data.decode()), +... }) +[(1+2j), Decimal('0.31')] +>>> +``` + Python standard library style names `dump`, `dumps`, `load`, `loads` are also available: - ``` python ->>> import umsgpack >>> umsgpack.dumps({u"compact": True, u"schema": 0}) -'\x82\xa7compact\xc3\xa6schema\x00' +b'\x82\xa7compact\xc3\xa6schema\x00' >>> umsgpack.loads(_) {u'compact': True, u'schema': 0} >>> @@ -101,9 +138,8 @@ available: ## More Information -See the [project page](https://github.com/vsergeev/u-msgpack-python) for more information on old specification compatibility mode, exceptions, behavior, and testing. +See the [project page](https://github.com/vsergeev/u-msgpack-python) for more information on options, exceptions, behavior, and testing. ## License u-msgpack-python is MIT licensed. See the included `LICENSE` file for more details. - diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..689fa54 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,5 @@ +[metadata] +description_file = README.md + +[bdist_wheel] +universal = True diff --git a/setup.py b/setup.py index fa314b0..1cf6809 100644 --- a/setup.py +++ b/setup.py @@ -1,13 +1,17 @@ -from distutils.core import setup +try: + from setuptools import setup +except ImportError: + from distutils.core import setup setup( name='u-msgpack-python', - version='2.1', + version='2.8.0', description='A portable, lightweight MessagePack serializer and deserializer written in pure Python.', author='vsergeev', - author_email='vsergeev at gmail', + author_email='v@sergeev.io', url='https://github.com/vsergeev/u-msgpack-python', - py_modules=['umsgpack'], + packages=['umsgpack'], + package_data={'umsgpack': ['*.pyi', 'py.typed']}, long_description="""u-msgpack-python is a lightweight `MessagePack `_ serializer and deserializer module written in pure Python, compatible with both Python 2 and Python 3, as well as CPython and PyPy implementations of Python. u-msgpack-python is fully compliant with the latest `MessagePack specification `_. In particular, it supports the new binary, UTF-8 string, and application-defined ext types. See https://github.com/vsergeev/u-msgpack-python for more information.""", classifiers=[ "Development Status :: 5 - Production/Stable", @@ -21,5 +25,4 @@ ], license='MIT', keywords='msgpack serialization deserialization', - ) - +) diff --git a/test_umsgpack.py b/test_umsgpack.py index 977b24b..1652d60 100644 --- a/test_umsgpack.py +++ b/test_umsgpack.py @@ -8,230 +8,382 @@ # $ pypy3 test_umsgpack.py # -import umsgpack import sys import struct import unittest +import datetime import io +from collections import OrderedDict, namedtuple + +import umsgpack single_test_vectors = [ # None - [ "nil", None, b"\xc0" ], + ["nil", None, b"\xc0"], # Booleans - [ "bool false", False, b"\xc2" ], - [ "bool true", True, b"\xc3" ], + ["bool false", False, b"\xc2"], + ["bool true", True, b"\xc3"], # + 7-bit uint - [ "7-bit uint", 0x00, b"\x00" ], - [ "7-bit uint", 0x10, b"\x10" ], - [ "7-bit uint", 0x7f, b"\x7f" ], + ["7-bit uint", 0x00, b"\x00"], + ["7-bit uint", 0x10, b"\x10"], + ["7-bit uint", 0x7f, b"\x7f"], # - 5-bit int - [ "5-bit sint", -1, b"\xff" ], - [ "5-bit sint", -16, b"\xf0" ], - [ "5-bit sint", -32, b"\xe0" ], + ["5-bit sint", -1, b"\xff"], + ["5-bit sint", -16, b"\xf0"], + ["5-bit sint", -32, b"\xe0"], # 8-bit uint - [ "8-bit uint", 0x80, b"\xcc\x80" ], - [ "8-bit uint", 0xf0, b"\xcc\xf0" ], - [ "8-bit uint", 0xff, b"\xcc\xff" ], + ["8-bit uint", 0x80, b"\xcc\x80"], + ["8-bit uint", 0xf0, b"\xcc\xf0"], + ["8-bit uint", 0xff, b"\xcc\xff"], # 16-bit uint - [ "16-bit uint", 0x100, b"\xcd\x01\x00" ], - [ "16-bit uint", 0x2000, b"\xcd\x20\x00" ], - [ "16-bit uint", 0xffff, b"\xcd\xff\xff" ], + ["16-bit uint", 0x100, b"\xcd\x01\x00"], + ["16-bit uint", 0x2000, b"\xcd\x20\x00"], + ["16-bit uint", 0xffff, b"\xcd\xff\xff"], # 32-bit uint - [ "32-bit uint", 0x10000, b"\xce\x00\x01\x00\x00" ], - [ "32-bit uint", 0x200000, b"\xce\x00\x20\x00\x00" ], - [ "32-bit uint", 0xffffffff, b"\xce\xff\xff\xff\xff" ], + ["32-bit uint", 0x10000, b"\xce\x00\x01\x00\x00"], + ["32-bit uint", 0x200000, b"\xce\x00\x20\x00\x00"], + ["32-bit uint", 0xffffffff, b"\xce\xff\xff\xff\xff"], # 64-bit uint - [ "64-bit uint", 0x100000000, b"\xcf" + b"\x00\x00\x00\x01" + b"\x00\x00\x00\x00" ], - [ "64-bit uint", 0x200000000000, b"\xcf" + b"\x00\x00\x20\x00" + b"\x00\x00\x00\x00" ], - [ "64-bit uint", 0xffffffffffffffff, b"\xcf" + b"\xff\xff\xff\xff" + b"\xff\xff\xff\xff" ], + ["64-bit uint", 0x100000000, b"\xcf\x00\x00\x00\x01\x00\x00\x00\x00"], + ["64-bit uint", 0x200000000000, b"\xcf\x00\x00\x20\x00\x00\x00\x00\x00"], + ["64-bit uint", 0xffffffffffffffff, b"\xcf\xff\xff\xff\xff\xff\xff\xff\xff"], # 8-bit int - [ "8-bit int", -33, b"\xd0\xdf" ], - [ "8-bit int", -100, b"\xd0\x9c" ], - [ "8-bit int", -128, b"\xd0\x80" ], + ["8-bit int", -33, b"\xd0\xdf"], + ["8-bit int", -100, b"\xd0\x9c"], + ["8-bit int", -128, b"\xd0\x80"], # 16-bit int - [ "16-bit int", -129, b"\xd1\xff\x7f" ], - [ "16-bit int", -2000, b"\xd1\xf8\x30" ], - [ "16-bit int", -32768, b"\xd1\x80\x00" ], + ["16-bit int", -129, b"\xd1\xff\x7f"], + ["16-bit int", -2000, b"\xd1\xf8\x30"], + ["16-bit int", -32768, b"\xd1\x80\x00"], # 32-bit int - [ "32-bit int", -32769, b"\xd2\xff\xff\x7f\xff" ], - [ "32-bit int", -1000000000, b"\xd2\xc4\x65\x36\x00" ], - [ "32-bit int", -2147483648, b"\xd2\x80\x00\x00\x00" ], + ["32-bit int", -32769, b"\xd2\xff\xff\x7f\xff"], + ["32-bit int", -1000000000, b"\xd2\xc4\x65\x36\x00"], + ["32-bit int", -2147483648, b"\xd2\x80\x00\x00\x00"], # 64-bit int - [ "64-bit int", -2147483649, b"\xd3" + b"\xff\xff\xff\xff" + b"\x7f\xff\xff\xff" ], - [ "64-bit int", -1000000000000000002, b"\xd3" + b"\xf2\x1f\x49\x4c" + b"\x58\x9b\xff\xfe" ], - [ "64-bit int", -9223372036854775808, b"\xd3" + b"\x80\x00\x00\x00" + b"\x00\x00\x00\x00" ], + ["64-bit int", -2147483649, b"\xd3\xff\xff\xff\xff\x7f\xff\xff\xff"], + ["64-bit int", -1000000000000000002, b"\xd3\xf2\x1f\x49\x4c\x58\x9b\xff\xfe"], + ["64-bit int", -9223372036854775808, b"\xd3\x80\x00\x00\x00\x00\x00\x00\x00"], # 64-bit float - [ "64-bit float", 0.0, b"\xcb" + b"\x00\x00\x00\x00" + b"\x00\x00\x00\x00" ], - [ "64-bit float", 2.5, b"\xcb" + b"\x40\x04\x00\x00" + b"\x00\x00\x00\x00" ], - [ "64-bit float", float(10**35), b"\xcb" + b"\x47\x33\x42\x61" + b"\x72\xc7\x4d\x82" ], + ["64-bit float", 0.0, b"\xcb\x00\x00\x00\x00\x00\x00\x00\x00"], + ["64-bit float", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00"], + ["64-bit float", float(10**35), b"\xcb\x47\x33\x42\x61\x72\xc7\x4d\x82"], # Fixstr String - [ "fix string", u"", b"\xa0" ], - [ "fix string", u"a", b"\xa1\x61" ], - [ "fix string", u"abc", b"\xa3\x61\x62\x63" ], - [ "fix string", u"a" * 31, b"\xbf" + b"\x61"*31 ], + ["fix string", u"", b"\xa0"], + ["fix string", u"a", b"\xa1\x61"], + ["fix string", u"abc", b"\xa3\x61\x62\x63"], + ["fix string", u"a" * 31, b"\xbf" + b"\x61" * 31], # 8-bit String - [ "8-bit string", u"b" * 32, b"\xd9\x20" + b"b" * 32 ], - [ "8-bit string", u"c" * 100, b"\xd9\x64" + b"c" * 100 ], - [ "8-bit string", u"d" * 255, b"\xd9\xff" + b"d" * 255 ], + ["8-bit string", u"b" * 32, b"\xd9\x20" + b"b" * 32], + ["8-bit string", u"c" * 100, b"\xd9\x64" + b"c" * 100], + ["8-bit string", u"d" * 255, b"\xd9\xff" + b"d" * 255], # 16-bit String - [ "16-bit string", u"b" * 256, b"\xda\x01\x00" + b"b" * 256 ], - [ "16-bit string", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535 ], + ["16-bit string", u"b" * 256, b"\xda\x01\x00" + b"b" * 256], + ["16-bit string", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535], # 32-bit String - [ "32-bit string", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536 ], + ["32-bit string", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536], # Wide character String - [ "wide char string", u"Allagbé", b"\xa8Allagb\xc3\xa9" ], - [ "wide char string", u"По оживлённым берегам", b"\xd9\x28\xd0\x9f\xd0\xbe\x20\xd0\xbe\xd0\xb6\xd0\xb8\xd0\xb2\xd0\xbb\xd1\x91\xd0\xbd\xd0\xbd\xd1\x8b\xd0\xbc\x20\xd0\xb1\xd0\xb5\xd1\x80\xd0\xb5\xd0\xb3\xd0\xb0\xd0\xbc" ], + ["wide char string", u"Allagbé", b"\xa8Allagb\xc3\xa9"], + ["wide char string", u"По оживлённым берегам", + b"\xd9\x28\xd0\x9f\xd0\xbe\x20\xd0\xbe\xd0\xb6\xd0\xb8\xd0\xb2\xd0\xbb\xd1\x91\xd0\xbd\xd0\xbd\xd1\x8b\xd0\xbc\x20\xd0\xb1\xd0\xb5\xd1\x80\xd0\xb5\xd0\xb3\xd0\xb0\xd0\xbc"], # 8-bit Binary - [ "8-bit binary", b"\x80" * 1, b"\xc4\x01" + b"\x80" * 1 ], - [ "8-bit binary", b"\x80" * 32, b"\xc4\x20" + b"\x80" * 32 ], - [ "8-bit binary", b"\x80" * 255, b"\xc4\xff" + b"\x80" * 255 ], + ["8-bit binary", b"\x80" * 1, b"\xc4\x01" + b"\x80" * 1], + ["8-bit binary", b"\x80" * 32, b"\xc4\x20" + b"\x80" * 32], + ["8-bit binary", b"\x80" * 255, b"\xc4\xff" + b"\x80" * 255], # 16-bit Binary - [ "16-bit binary", b"\x80" * 256, b"\xc5\x01\x00" + b"\x80" * 256 ], + ["16-bit binary", b"\x80" * 256, b"\xc5\x01\x00" + b"\x80" * 256], # 32-bit Binary - [ "32-bit binary", b"\x80" * 65536, b"\xc6\x00\x01\x00\x00" + b"\x80" * 65536 ], + ["32-bit binary", b"\x80" * 65536, b"\xc6\x00\x01\x00\x00" + b"\x80" * 65536], # Fixext 1 - [ "fixext 1", umsgpack.Ext(0x05, b"\x80"*1), b"\xd4\x05" + b"\x80"*1 ], + ["fixext 1", umsgpack.Ext(0x05, b"\x80" * 1), b"\xd4\x05" + b"\x80" * 1], # Fixext 2 - [ "fixext 2", umsgpack.Ext(0x05, b"\x80"*2), b"\xd5\x05" + b"\x80"*2 ], + ["fixext 2", umsgpack.Ext(0x05, b"\x80" * 2), b"\xd5\x05" + b"\x80" * 2], # Fixext 4 - [ "fixext 4", umsgpack.Ext(0x05, b"\x80"*4), b"\xd6\x05" + b"\x80"*4 ], + ["fixext 4", umsgpack.Ext(0x05, b"\x80" * 4), b"\xd6\x05" + b"\x80" * 4], # Fixext 8 - [ "fixext 8", umsgpack.Ext(0x05, b"\x80"*8), b"\xd7\x05" + b"\x80"*8 ], + ["fixext 8", umsgpack.Ext(0x05, b"\x80" * 8), b"\xd7\x05" + b"\x80" * 8], # Fixext 16 - [ "fixext 16", umsgpack.Ext(0x05, b"\x80"*16), b"\xd8\x05" + b"\x80"*16 ], + ["fixext 16", umsgpack.Ext(0x05, b"\x80" * 16), + b"\xd8\x05" + b"\x80" * 16], # 8-bit Ext - [ "8-bit ext", umsgpack.Ext(0x05, b"\x80"*255), b"\xc7\xff\x05" + b"\x80"*255 ], + ["8-bit ext", umsgpack.Ext(0x05, b"\x80" * 255), + b"\xc7\xff\x05" + b"\x80" * 255], # 16-bit Ext - [ "16-bit ext", umsgpack.Ext(0x05, b"\x80"*256), b"\xc8\x01\x00\x05" + b"\x80"*256 ], + ["16-bit ext", umsgpack.Ext(0x05, b"\x80" * 256), + b"\xc8\x01\x00\x05" + b"\x80" * 256], # 32-bit Ext - [ "32-bit ext", umsgpack.Ext(0x05, b"\x80"*65536), b"\xc9\x00\x01\x00\x00\x05" + b"\x80"*65536 ], + ["32-bit ext", umsgpack.Ext(0x05, b"\x80" * 65536), + b"\xc9\x00\x01\x00\x00\x05" + b"\x80" * 65536], # Empty Array - [ "empty array", [], b"\x90" ], + ["empty array", [], b"\x90"], # Empty Map - [ "empty map", {}, b"\x80" ], + ["empty map", {}, b"\x80"], + # 32-bit Timestamp + ["32-bit timestamp", datetime.datetime(1970, 1, 1, 0, 0, 0, 0, umsgpack._utc_tzinfo), + b"\xd6\xff\x00\x00\x00\x00"], + ["32-bit timestamp", datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xd6\xff\x38\x6d\xd1\x4e"], + # 64-bit Timestamp + ["64-bit timestamp", datetime.datetime(2000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x4b\x51\x40\x38\x6d\xd1\x4e"], + ["64-bit timestamp", datetime.datetime(2200, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x00\x00\x01\xb0\x9e\xa6\xce"], + ["64-bit timestamp", datetime.datetime(2200, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x4b\x51\x41\xb0\x9e\xa6\xce"], + # 96-bit Timestamp + ["96-bit timestamp", datetime.datetime(1900, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x00\x00\x00\xff\xff\xff\xff\x7c\x56\x0f\x4e"], + ["96-bit timestamp", datetime.datetime(1900, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x12\xd4\x50\xff\xff\xff\xff\x7c\x56\x0f\x4e"], + ["96-bit timestamp", datetime.datetime(3000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x00\x00\x00\x00\x00\x00\x07\x91\x5f\x59\xce"], + ["96-bit timestamp", datetime.datetime(3000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x12\xd4\x50\x00\x00\x00\x07\x91\x5f\x59\xce"], ] composite_test_vectors = [ # Fix Array - [ "fix array", [ 5, u"abc", True ], b"\x93\x05\xa3\x61\x62\x63\xc3" ], + ["fix array", [5, u"abc", True], + b"\x93\x05\xa3\x61\x62\x63\xc3"], # 16-bit Array - [ "16-bit array", [ 0x05 ]*16, b"\xdc\x00\x10" + b"\x05"*16 ], - [ "16-bit array", [ 0x05 ]*65535, b"\xdc\xff\xff" + b"\x05"*65535 ], + ["16-bit array", [0x05] * 16, + b"\xdc\x00\x10" + b"\x05" * 16], + ["16-bit array", [0x05] * 65535, + b"\xdc\xff\xff" + b"\x05" * 65535], # 32-bit Array - [ "32-bit array", [ 0x05 ]*65536, b"\xdd\x00\x01\x00\x00" + b"\x05"*65536 ], + ["32-bit array", [0x05] * 65536, + b"\xdd\x00\x01\x00\x00" + b"\x05" * 65536], # Fix Map - [ "fix map", { 1: True, 2: u"abc", 3: b"\x80" }, b"\x83\x01\xc3\x02\xa3\x61\x62\x63\x03\xc4\x01\x80" ], - [ "fix map", { u"abc" : 5 }, b"\x81\xa3\x61\x62\x63\x05" ], - [ "fix map", { b"\x80" : 0xffff }, b"\x81\xc4\x01\x80\xcd\xff\xff" ], - [ "fix map", { True : None }, b"\x81\xc3\xc0" ], + ["fix map", OrderedDict([(1, True), (2, u"abc"), (3, b"\x80")]), + b"\x83\x01\xc3\x02\xa3\x61\x62\x63\x03\xc4\x01\x80"], + ["fix map", {u"abc": 5}, + b"\x81\xa3\x61\x62\x63\x05"], + ["fix map", {b"\x80": 0xffff}, + b"\x81\xc4\x01\x80\xcd\xff\xff"], + ["fix map", {True: None}, + b"\x81\xc3\xc0"], # 16-bit Map - [ "16-bit map", dict([(k, 0x05) for k in range(16)]), b"\xde\x00\x10" + b"".join( [ struct.pack("B", i) + b"\x05" for i in range(16) ] ) ], - [ "16-bit map", dict([(k, 0x05) for k in range(6000)]), b"\xde\x17\x70" + b"".join([ struct.pack("B", i) + b"\x05" for i in range(128)]) + b"".join([ b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + b"".join([ b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)]) ], + ["16-bit map", OrderedDict([(k, 0x05) for k in range(16)]), + b"\xde\x00\x10" + b"".join([struct.pack("B", i) + b"\x05" for i in range(16)])], + ["16-bit map", OrderedDict([(k, 0x05) for k in range(6000)]), + b"\xde\x17\x70" + b"".join([struct.pack("B", i) + b"\x05" for i in range(128)]) + + b"".join([b"\xcc" + struct.pack("B", i) + b"\x05" for i in range(128, 256)]) + + b"".join([b"\xcd" + struct.pack(">H", i) + b"\x05" for i in range(256, 6000)])], # Complex Array - [ "complex array", [ True, 0x01, umsgpack.Ext(0x03, b"foo"), 0xff, { 1: False, 2: u"abc" }, b"\x80", [ 1, 2, 3], u"abc" ], b"\x98\xc3\x01\xc7\x03\x03\x66\x6f\x6f\xcc\xff\x82\x01\xc2\x02\xa3\x61\x62\x63\xc4\x01\x80\x93\x01\x02\x03\xa3\x61\x62\x63" ], + ["complex array", [True, 0x01, umsgpack.Ext(0x03, b"foo"), 0xff, + OrderedDict([(1, False), (2, u"abc")]), b"\x80", + [1, 2, 3], u"abc"], + b"\x98\xc3\x01\xc7\x03\x03\x66\x6f\x6f\xcc\xff\x82\x01\xc2\x02\xa3\x61\x62\x63\xc4\x01\x80\x93\x01\x02\x03\xa3\x61\x62\x63"], # Complex Map - [ "complex map", { 1 : [{1: 2, 3: 4}, {}], 2: 1, 3: [False, u"def"], 4: {0x100000000: u"a", 0xffffffff: u"b"}}, b"\x84\x01\x92\x82\x01\x02\x03\x04\x80\x02\x01\x03\x92\xc2\xa3\x64\x65\x66\x04\x82\xcf\x00\x00\x00\x01\x00\x00\x00\x00\xa1\x61\xce\xff\xff\xff\xff\xa1\x62" ], + ["complex map", OrderedDict([(1, [OrderedDict([(1, 2), (3, 4)]), {}]), + (2, 1), (3, [False, u"def"]), + (4, OrderedDict([(0x100000000, u"a"), + (0xffffffff, u"b")]))]), + b"\x84\x01\x92\x82\x01\x02\x03\x04\x80\x02\x01\x03\x92\xc2\xa3\x64\x65\x66\x04\x82\xcf\x00\x00\x00\x01\x00\x00\x00\x00\xa1\x61\xce\xff\xff\xff\xff\xa1\x62"], # Map with Tuple Keys - [ "map with tuple keys", {(u"foo", False, 3) : True, (3e6, -5): u"def"}, b"\x82\x92\xcb\x41\x46\xe3\x60\x00\x00\x00\x00\xfb\xa3\x64\x65\x66\x93\xa3\x66\x6f\x6f\xc2\x03\xc3" ], + ["map with tuple keys", OrderedDict([((u"foo", False, 3), True), + ((3e6, -5), u"def")]), + b"\x82\x93\xa3\x66\x6f\x6f\xc2\x03\xc3\x92\xcb\x41\x46\xe3\x60\x00\x00\x00\x00\xfb\xa3\x64\x65\x66"], # Map with Complex Tuple Keys - [ "map with complex tuple keys", {(u"foo", (1,2,3), 3) : -5}, b"\x81\x93\xa3\x66\x6f\x6f\x93\x01\x02\x03\x03\xfb" ] + ["map with complex tuple keys", {(u"foo", (1, 2, 3), 3): -5}, + b"\x81\x93\xa3\x66\x6f\x6f\x93\x01\x02\x03\x03\xfb"] ] pack_exception_test_vectors = [ # Unsupported type exception - [ "unsupported type", set([1,2,3]), umsgpack.UnsupportedTypeException ], - [ "unsupported type", -2**(64-1)-1, umsgpack.UnsupportedTypeException ], - [ "unsupported type", 2**64, umsgpack.UnsupportedTypeException ], + ["unsupported type", set([1, 2, 3]), umsgpack.UnsupportedTypeException], + ["unsupported type", -2**(64 - 1) - 1, umsgpack.UnsupportedTypeException], + ["unsupported type", 2**64, umsgpack.UnsupportedTypeException], ] unpack_exception_test_vectors = [ # Type errors - [ "type error unpack unicode string", u"\x01", TypeError ], - [ "type error unpack boolean", True, TypeError ], + ["type error unpack unicode string", u"\x01", TypeError], + ["type error unpack boolean", True, TypeError], # Insufficient data to unpack object - [ "insufficient data 8-bit uint", b"\xcc", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit uint", b"\xcd\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit uint", b"\xce\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 64-bit uint", b"\xcf\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit int", b"\xd0", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit int", b"\xd1\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit int", b"\xd2\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 64-bit int", b"\xd3\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit float", b"\xca\xff", umsgpack.InsufficientDataException ], - [ "insufficient data 64-bit float", b"\xcb\xff", umsgpack.InsufficientDataException ], - [ "insufficient data fixstr", b"\xa1", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit string", b"\xd9", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit string", b"\xd9\x01", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit string", b"\xda\x01\x00", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit string", b"\xdb\x00\x01\x00\x00", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit binary", b"\xc4", umsgpack.InsufficientDataException ], - [ "insufficient data 8-bit binary", b"\xc4\x01", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit binary", b"\xc5\x01\x00", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit binary", b"\xc6\x00\x01\x00\x00", umsgpack.InsufficientDataException ], - [ "insufficient data fixarray", b"\x91", umsgpack.InsufficientDataException ], - [ "insufficient data fixarray", b"\x92\xc2", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit array", b"\xdc\x00\xf0\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit array", b"\xdd\x00\x01\x00\x00\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data fixmap", b"\x81", umsgpack.InsufficientDataException ], - [ "insufficient data fixmap", b"\x82\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data 16-bit map", b"\xde\x00\xf0\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data 32-bit map", b"\xdf\x00\x01\x00\x00\xc2\xc3", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 1", b"\xd4", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 1", b"\xd4\x05", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 2", b"\xd5\x05\x01", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 4", b"\xd6\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 8", b"\xd7\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data fixext 16", b"\xd8\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data ext 8-bit", b"\xc7\x05\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data ext 16-bit", b"\xc8\x01\x00\x05\x01\x02\x03", umsgpack.InsufficientDataException ], - [ "insufficient data ext 32-bit", b"\xc9\x00\x01\x00\x00\x05\x01\x02\x03", umsgpack.InsufficientDataException ], + ["insufficient data 8-bit uint", b"\xcc", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit uint", b"\xcd\xff", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit uint", b"\xce\xff", + umsgpack.InsufficientDataException], + ["insufficient data 64-bit uint", b"\xcf\xff", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit int", b"\xd0", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit int", b"\xd1\xff", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit int", b"\xd2\xff", + umsgpack.InsufficientDataException], + ["insufficient data 64-bit int", b"\xd3\xff", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit float", b"\xca\xff", + umsgpack.InsufficientDataException], + ["insufficient data 64-bit float", b"\xcb\xff", + umsgpack.InsufficientDataException], + ["insufficient data fixstr", b"\xa1", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit string", b"\xd9", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit string", b"\xd9\x01", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit string", b"\xda\x01\x00", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit string", b"\xdb\x00\x01\x00\x00", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit binary", b"\xc4", + umsgpack.InsufficientDataException], + ["insufficient data 8-bit binary", b"\xc4\x01", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit binary", b"\xc5\x01\x00", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit binary", b"\xc6\x00\x01\x00\x00", + umsgpack.InsufficientDataException], + ["insufficient data fixarray", b"\x91", + umsgpack.InsufficientDataException], + ["insufficient data fixarray", b"\x92\xc2", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit array", b"\xdc\x00\xf0\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit array", b"\xdd\x00\x01\x00\x00\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data fixmap", b"\x81", + umsgpack.InsufficientDataException], + ["insufficient data fixmap", b"\x82\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data 16-bit map", b"\xde\x00\xf0\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data 32-bit map", b"\xdf\x00\x01\x00\x00\xc2\xc3", + umsgpack.InsufficientDataException], + ["insufficient data fixext 1", b"\xd4", + umsgpack.InsufficientDataException], + ["insufficient data fixext 1", b"\xd4\x05", + umsgpack.InsufficientDataException], + ["insufficient data fixext 2", b"\xd5\x05\x01", + umsgpack.InsufficientDataException], + ["insufficient data fixext 4", b"\xd6\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data fixext 8", b"\xd7\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data fixext 16", b"\xd8\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data ext 8-bit", b"\xc7\x05\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data ext 16-bit", b"\xc8\x01\x00\x05\x01\x02\x03", + umsgpack.InsufficientDataException], + ["insufficient data ext 32-bit", b"\xc9\x00\x01\x00\x00\x05\x01\x02\x03", + umsgpack.InsufficientDataException], # Unhashable key { 1 : True, { 1 : 1 } : False } - [ "unhashable key", b"\x82\x01\xc3\x81\x01\x01\xc2", umsgpack.UnhashableKeyException ], + ["unhashable key", b"\x82\x01\xc3\x81\x01\x01\xc2", + umsgpack.UnhashableKeyException], # Unhashable key { [ 1, 2, {} ] : True } - [ "unhashable key", b"\x81\x93\x01\x02\x80\xc3", umsgpack.UnhashableKeyException ], + ["unhashable key", b"\x81\x93\x01\x02\x80\xc3", + umsgpack.UnhashableKeyException], # Key duplicate { 1 : True, 1 : False } - [ "duplicate key", b"\x82\x01\xc3\x01\xc2", umsgpack.DuplicateKeyException ], + ["duplicate key", b"\x82\x01\xc3\x01\xc2", + umsgpack.DuplicateKeyException], # Reserved code (0xc1) - [ "reserved code", b"\xc1", umsgpack.ReservedCodeException ], + ["reserved code", b"\xc1", + umsgpack.ReservedCodeException], + # Unsupported timestamp (unsupported data length) + ["unsupported timestamp", b"\xc7\x02\xff\xaa\xbb", + umsgpack.UnsupportedTimestampException], # Invalid string (non utf-8) - [ "invalid string", b"\xa1\x80", umsgpack.InvalidStringException ], + ["invalid string", b"\xa1\x80", + umsgpack.InvalidStringException], ] compatibility_test_vectors = [ # Fix Raw - [ "fix raw", b"", b"\xa0" ], - [ "fix raw", u"", b"\xa0" ], - [ "fix raw", b"a", b"\xa1\x61" ], - [ "fix raw", u"a", b"\xa1\x61" ], - [ "fix raw", b"abc", b"\xa3\x61\x62\x63" ], - [ "fix raw", u"abc", b"\xa3\x61\x62\x63" ], - [ "fix raw", b"a" * 31, b"\xbf" + b"\x61"*31 ], - [ "fix raw", u"a" * 31, b"\xbf" + b"\x61"*31 ], + ["fix raw", b"", b"\xa0"], + ["fix raw", u"", b"\xa0"], + ["fix raw", b"a", b"\xa1\x61"], + ["fix raw", u"a", b"\xa1\x61"], + ["fix raw", b"abc", b"\xa3\x61\x62\x63"], + ["fix raw", u"abc", b"\xa3\x61\x62\x63"], + ["fix raw", b"a" * 31, b"\xbf" + b"\x61" * 31], + ["fix raw", u"a" * 31, b"\xbf" + b"\x61" * 31], # 16-bit Raw - [ "16-bit raw", u"b" * 32, b"\xda\x00\x20" + b"b" * 32 ], - [ "16-bit raw", b"b" * 32, b"\xda\x00\x20" + b"b" * 32 ], - [ "16-bit raw", u"b" * 256, b"\xda\x01\x00" + b"b" * 256 ], - [ "16-bit raw", b"b" * 256, b"\xda\x01\x00" + b"b" * 256 ], - [ "16-bit raw", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535 ], - [ "16-bit raw", b"c" * 65535, b"\xda\xff\xff" + b"c" * 65535 ], + ["16-bit raw", u"b" * 32, b"\xda\x00\x20" + b"b" * 32], + ["16-bit raw", b"b" * 32, b"\xda\x00\x20" + b"b" * 32], + ["16-bit raw", u"b" * 256, b"\xda\x01\x00" + b"b" * 256], + ["16-bit raw", b"b" * 256, b"\xda\x01\x00" + b"b" * 256], + ["16-bit raw", u"c" * 65535, b"\xda\xff\xff" + b"c" * 65535], + ["16-bit raw", b"c" * 65535, b"\xda\xff\xff" + b"c" * 65535], # 32-bit Raw - [ "32-bit raw", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536 ], - [ "32-bit raw", b"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536 ], + ["32-bit raw", u"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536], + ["32-bit raw", b"b" * 65536, b"\xdb\x00\x01\x00\x00" + b"b" * 65536], +] + +float_precision_test_vectors = [ + ["float precision single", 2.5, b"\xca\x40\x20\x00\x00"], + ["float precision double", 2.5, b"\xcb\x40\x04\x00\x00\x00\x00\x00\x00"], +] + +tuple_test_vectors = [ + ["nested array", [0x01, [b"\x80", [[u"a", u"b", u"c"], True]]], + b"\x92\x01\x92\xc4\x01\x80\x92\x93\xa1a\xa1b\xa1c\xc3", + (0x01, (b"\x80", ((u"a", u"b", u"c"), True)))], +] + +naive_timestamp_test_vectors = [ + ["32-bit timestamp (naive)", datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo), + b"\xd6\xff\x38\x6d\xd1\x4e", + datetime.datetime(2000, 1, 1, 10, 5, 2, 0, umsgpack._utc_tzinfo)], + ["64-bit timestamp (naive)", datetime.datetime(2200, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xd7\xff\x00\x4b\x51\x41\xb0\x9e\xa6\xce", + datetime.datetime(2200, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo)], + ["96-bit timestamp (naive)", datetime.datetime(3000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo), + b"\xc7\x0c\xff\x00\x12\xd4\x50\x00\x00\x00\x07\x91\x5f\x59\xce", + datetime.datetime(3000, 1, 1, 10, 5, 2, 1234, umsgpack._utc_tzinfo)], +] + +CustomType = namedtuple('CustomType', ['x', 'y', 'z']) + +ext_handlers = { + complex: lambda obj: umsgpack.Ext(0x20, struct.pack(" unpackb -> {int,long} if isinstance(obj, int) or isinstance(obj, long): - self.assertTrue(isinstance(unpacked, int) or isinstance(unpacked, long)) + self.assertTrue(isinstance(unpacked, int) or + isinstance(unpacked, long)) else: self.assertTrue(isinstance(unpacked, type(obj))) # In Python3, we only have the int integer type @@ -289,12 +452,15 @@ def test_unpack_single(self): def test_unpack_composite(self): for (name, obj, data) in composite_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.unpackb(data), obj) def test_unpack_exceptions(self): for (name, data, exception) in unpack_exception_test_vectors: - print("\tTesting %s" % name) + print("\tTesting {:s}".format(name)) + with self.assertRaises(exception): umsgpack.unpackb(data) @@ -303,7 +469,9 @@ def test_pack_compatibility(self): for (name, obj, data) in compatibility_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + self.assertEqual(umsgpack.packb(obj), data) umsgpack.compatibility = False @@ -313,7 +481,9 @@ def test_unpack_compatibility(self): for (name, obj, data) in compatibility_test_vectors: obj_repr = repr(obj) - print("\tTesting %s: object %s" % (name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + unpacked = umsgpack.unpackb(data) # Encoded raw should always unpack to bytes in compatibility mode, @@ -330,39 +500,305 @@ def test_unpack_compatibility(self): umsgpack.compatibility = False - def test_ext_exceptions(self): - with self.assertRaises(TypeError): - _ = umsgpack.Ext(-1, b"") + def test_unpack_invalid_string(self): + # Use last unpack exception test vector (an invalid string) + (_, data, _) = unpack_exception_test_vectors[-1] + + obj = umsgpack.unpackb(data, allow_invalid_utf8=True) + self.assertTrue(isinstance(obj, umsgpack.InvalidString)) + self.assertEqual(obj, b"\x80") + + def test_unpack_ordered_dict(self): + # Use last composite test vector (a map) + (_, obj, data) = composite_test_vectors[-1] + + # Unpack with default options (unordered dict) + unpacked = umsgpack.unpackb(data) + self.assertTrue(isinstance(unpacked, dict)) + + # Unpack with unordered dict + unpacked = umsgpack.unpackb(data, use_ordered_dict=False) + self.assertTrue(isinstance(unpacked, dict)) + + # Unpack with ordered dict + unpacked = umsgpack.unpackb(data, use_ordered_dict=True) + self.assertTrue(isinstance(unpacked, OrderedDict)) + self.assertEqual(unpacked, obj) + + def test_unpack_tuple(self): + # Use tuple test vector + (_, obj, data, obj_tuple) = tuple_test_vectors[0] + # Unpack with default options (list) + self.assertEqual(umsgpack.unpackb(data), obj) + + # Unpack with use_tuple=False (list) + self.assertEqual(umsgpack.unpackb(data, use_tuple=False), obj) + + # Unpack with use_tuple=True (tuple) + self.assertEqual(umsgpack.unpackb(data, use_tuple=True), obj_tuple) + + def test_ext_exceptions(self): + # Test invalid Ext type type with self.assertRaises(TypeError): - _ = umsgpack.Ext(128, b"") + _ = umsgpack.Ext(5.0, b"") + # Test invalid data type with self.assertRaises(TypeError): _ = umsgpack.Ext(0, u"unicode string") + # Test out of range Ext type value + with self.assertRaises(ValueError): + _ = umsgpack.Ext(-129, b"data") + with self.assertRaises(ValueError): + _ = umsgpack.Ext(128, b"data") + + def test_pack_ext_handler(self): + for (name, obj, data) in ext_handlers_test_vectors: + obj_repr = repr(obj) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj, ext_handlers=ext_handlers) + self.assertEqual(packed, data) + + def test_unpack_ext_handler(self): + for (name, obj, data) in ext_handlers_test_vectors: + obj_repr = repr(obj) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + unpacked = umsgpack.unpackb(data, ext_handlers=ext_handlers) + self.assertEqual(unpacked, obj) + + def test_pack_force_float_precision(self): + for ((name, obj, data), precision) in zip(float_precision_test_vectors, ["single", "double"]): + obj_repr = repr(obj) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj, force_float_precision=precision) + self.assertEqual(packed, data) + + def test_pack_naive_timestamp(self): + for (name, obj, data, _) in naive_timestamp_test_vectors: + obj_repr = repr(obj) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj) + self.assertEqual(packed, data) + + def test_unpack_naive_timestamp(self): + for (name, _, data, obj) in naive_timestamp_test_vectors: + obj_repr = repr(obj) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + unpacked = umsgpack.unpackb(data) + self.assertEqual(unpacked, obj) + + def test_pack_ext_override(self): + # Test overridden packing of datetime.datetime + (name, obj, data) = override_ext_handlers_test_vectors[0] + obj_repr = repr(obj) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + packed = umsgpack.packb(obj, ext_handlers=override_ext_handlers) + self.assertEqual(packed, data) + + def test_unpack_ext_override(self): + # Test overridden unpacking of Ext type -1 + (name, obj, data) = override_ext_handlers_test_vectors[1] + obj_repr = repr(obj) + print("\tTesting {:s}: object {:s}".format( + name, obj_repr if len(obj_repr) < 24 else obj_repr[0:24] + "...")) + + unpacked = umsgpack.unpackb(data, ext_handlers=override_ext_handlers) + self.assertEqual(unpacked, obj) + + def test_ext_handlers_subclass(self): + class Rectangle: + def __init__(self, length, width): + self.length = length + self.width = width + + def __eq__(self, other): + return self.length == other.length and self.width == other.width + + class Square(Rectangle): + def __init__(self, width): + Rectangle.__init__(self, width, width) + + # Test pack (packs base class) + packed = umsgpack.packb(Square(5), ext_handlers={ + Rectangle: lambda obj: umsgpack.Ext(0x10, umsgpack.packb([obj.length, obj.width])), + }) + self.assertEqual(packed, b"\xc7\x03\x10\x92\x05\x05") + + # Test unpack (unpacks base class) + unpacked = umsgpack.unpackb(packed, ext_handlers={ + 0x10: lambda ext: Rectangle(*umsgpack.unpackb(ext.data)), + }) + self.assertEqual(unpacked, Rectangle(5, 5)) + + def test_ext_serializable(self): + # Register test class + @umsgpack.ext_serializable(0x20) + class CustomComplex: + def __init__(self, real, imag): + self.real = real + self.imag = imag + + def __eq__(self, other): + return self.real == other.real and self.imag == other.imag + + def packb(self): + return struct.pack(">> foo = umsgpack.Ext(0x05, b"\x01\x02\x03") - >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) - '\x82\xa7awesome\xc3\xadspecial stuff\xc7\x03\x05\x01\x02\x03' - >>> bar = umsgpack.unpackb(_) - >>> print(bar["special stuff"]) - Ext Object (Type: 0x05, Data: 01 02 03) - >>> - """ - # Application ext type should be 0 <= type <= 127 - if not isinstance(type, int) or not (type >= 0 and type <= 127): - raise TypeError("ext type out of range") - # Check data is type bytes - elif sys.version_info[0] == 3 and not isinstance(data, bytes): - raise TypeError("ext data is not type \'bytes\'") - elif sys.version_info[0] == 2 and not isinstance(data, str): - raise TypeError("ext data is not type \'str\'") - self.type = type - self.data = data - - def __eq__(self, other): - """ - Compare this Ext object with another for equality. - """ - return (isinstance(other, self.__class__) and - self.type == other.type and - self.data == other.data) - - def __ne__(self, other): - """ - Compare this Ext object with another for inequality. - """ - return not self.__eq__(other) - - def __str__(self): - """ - String representation of this Ext object. - """ - s = "Ext Object (Type: 0x%02x, Data: " % self.type - for i in range(min(len(self.data), 8)): - if i > 0: - s += " " - if isinstance(self.data[i], int): - s += "%02x" % (self.data[i]) - else: - s += "%02x" % ord(self.data[i]) - if len(self.data) > 8: - s += " ..." - s += ")" - return s - -################################################################################ -### Exceptions -################################################################################ - -# Base Exception classes -class PackException(Exception): - "Base class for exceptions encountered during packing." - pass -class UnpackException(Exception): - "Base class for exceptions encountered during unpacking." - pass - -# Packing error -class UnsupportedTypeException(PackException): - "Object type not supported for packing." - pass - -# Unpacking error -class InsufficientDataException(UnpackException): - "Insufficient data to unpack the encoded object." - pass -class InvalidStringException(UnpackException): - "Invalid UTF-8 string encountered during unpacking." - pass -class ReservedCodeException(UnpackException): - "Reserved code encountered during unpacking." - pass -class UnhashableKeyException(UnpackException): - """ - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - """ - pass -class DuplicateKeyException(UnpackException): - "Duplicate key encountered during map unpacking." - pass - -# Backwards compatibility -KeyNotPrimitiveException = UnhashableKeyException -KeyDuplicateException = DuplicateKeyException - -################################################################################ -### Exported Functions and Globals -################################################################################ - -# Exported functions and variables, set up in __init() -pack = None -packb = None -unpack = None -unpackb = None -dump = None -dumps = None -load = None -loads = None - -compatibility = False -""" -Compatibility mode boolean. - -When compatibility mode is enabled, u-msgpack-python will serialize both -unicode strings and bytes into the old "raw" msgpack type, and deserialize the -"raw" msgpack type into bytes. This provides backwards compatibility with the -old MessagePack specification. - -Example: ->>> umsgpack.compatibility = True ->>> ->>> umsgpack.packb([u"some string", b"some bytes"]) -b'\x92\xabsome string\xaasome bytes' ->>> umsgpack.unpackb(_) -[b'some string', b'some bytes'] ->>> -""" - -################################################################################ -### Packing -################################################################################ - -# You may notice struct.pack("B", obj) instead of the simpler chr(obj) in the -# code below. This is to allow for seamless Python 2 and 3 compatibility, as -# chr(obj) has a str return type instead of bytes in Python 3, and -# struct.pack(...) has the right return type in both versions. - -def _pack_integer(obj, fp): - if obj < 0: - if obj >= -32: - fp.write(struct.pack("b", obj)) - elif obj >= -2**(8-1): - fp.write(b"\xd0" + struct.pack("b", obj)) - elif obj >= -2**(16-1): - fp.write(b"\xd1" + struct.pack(">h", obj)) - elif obj >= -2**(32-1): - fp.write(b"\xd2" + struct.pack(">i", obj)) - elif obj >= -2**(64-1): - fp.write(b"\xd3" + struct.pack(">q", obj)) - else: - raise UnsupportedTypeException("huge signed int") - else: - if obj <= 127: - fp.write(struct.pack("B", obj)) - elif obj <= 2**8-1: - fp.write(b"\xcc" + struct.pack("B", obj)) - elif obj <= 2**16-1: - fp.write(b"\xcd" + struct.pack(">H", obj)) - elif obj <= 2**32-1: - fp.write(b"\xce" + struct.pack(">I", obj)) - elif obj <= 2**64-1: - fp.write(b"\xcf" + struct.pack(">Q", obj)) - else: - raise UnsupportedTypeException("huge unsigned int") - -def _pack_nil(obj, fp): - fp.write(b"\xc0") - -def _pack_boolean(obj, fp): - fp.write(b"\xc3" if obj else b"\xc2") - -def _pack_float(obj, fp): - if _float_size == 64: - fp.write(b"\xcb" + struct.pack(">d", obj)) - else: - fp.write(b"\xca" + struct.pack(">f", obj)) - -def _pack_string(obj, fp): - obj = obj.encode('utf-8') - if len(obj) <= 31: - fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**8-1: - fp.write(b"\xd9" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16-1: - fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32-1: - fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) - else: - raise UnsupportedTypeException("huge string") - -def _pack_binary(obj, fp): - if len(obj) <= 2**8-1: - fp.write(b"\xc4" + struct.pack("B", len(obj)) + obj) - elif len(obj) <= 2**16-1: - fp.write(b"\xc5" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32-1: - fp.write(b"\xc6" + struct.pack(">I", len(obj)) + obj) - else: - raise UnsupportedTypeException("huge binary string") - -def _pack_oldspec_raw(obj, fp): - if len(obj) <= 31: - fp.write(struct.pack("B", 0xa0 | len(obj)) + obj) - elif len(obj) <= 2**16-1: - fp.write(b"\xda" + struct.pack(">H", len(obj)) + obj) - elif len(obj) <= 2**32-1: - fp.write(b"\xdb" + struct.pack(">I", len(obj)) + obj) - else: - raise UnsupportedTypeException("huge raw string") - -def _pack_ext(obj, fp): - if len(obj.data) == 1: - fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 2: - fp.write(b"\xd5" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 4: - fp.write(b"\xd6" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 8: - fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) == 16: - fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**8-1: - fp.write(b"\xc7" + struct.pack("BB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**16-1: - fp.write(b"\xc8" + struct.pack(">HB", len(obj.data), obj.type & 0xff) + obj.data) - elif len(obj.data) <= 2**32-1: - fp.write(b"\xc9" + struct.pack(">IB", len(obj.data), obj.type & 0xff) + obj.data) - else: - raise UnsupportedTypeException("huge ext data") - -def _pack_array(obj, fp): - if len(obj) <= 15: - fp.write(struct.pack("B", 0x90 | len(obj))) - elif len(obj) <= 2**16-1: - fp.write(b"\xdc" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32-1: - fp.write(b"\xdd" + struct.pack(">I", len(obj))) - else: - raise UnsupportedTypeException("huge array") - - for e in obj: - pack(e, fp) - -def _pack_map(obj, fp): - if len(obj) <= 15: - fp.write(struct.pack("B", 0x80 | len(obj))) - elif len(obj) <= 2**16-1: - fp.write(b"\xde" + struct.pack(">H", len(obj))) - elif len(obj) <= 2**32-1: - fp.write(b"\xdf" + struct.pack(">I", len(obj))) - else: - raise UnsupportedTypeException("huge array") - - for k,v in obj.items(): - pack(k, fp) - pack(v, fp) - -######################################## - -# Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type -def _pack2(obj, fp): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - fp: a .write()-supporting file-like object - - Returns: - None. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> f = open('test.bin', 'wb') - >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) - >>> - """ - - global compatibility - - if obj is None: - _pack_nil(obj, fp) - elif isinstance(obj, bool): - _pack_boolean(obj, fp) - elif isinstance(obj, int) or isinstance(obj, long): - _pack_integer(obj, fp) - elif isinstance(obj, float): - _pack_float(obj, fp) - elif compatibility and isinstance(obj, unicode): - _pack_oldspec_raw(bytes(obj), fp) - elif compatibility and isinstance(obj, bytes): - _pack_oldspec_raw(obj, fp) - elif isinstance(obj, unicode): - _pack_string(obj, fp) - elif isinstance(obj, str): - _pack_binary(obj, fp) - elif isinstance(obj, list) or isinstance(obj, tuple): - _pack_array(obj, fp) - elif isinstance(obj, dict): - _pack_map(obj, fp) - elif isinstance(obj, Ext): - _pack_ext(obj, fp) - else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) - -# Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type -def _pack3(obj, fp): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - fp: a .write()-supporting file-like object - - Returns: - None. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> f = open('test.bin', 'wb') - >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) - >>> - """ - global compatibility - - if obj is None: - _pack_nil(obj, fp) - elif isinstance(obj, bool): - _pack_boolean(obj, fp) - elif isinstance(obj, int): - _pack_integer(obj, fp) - elif isinstance(obj, float): - _pack_float(obj, fp) - elif compatibility and isinstance(obj, str): - _pack_oldspec_raw(obj.encode('utf-8'), fp) - elif compatibility and isinstance(obj, bytes): - _pack_oldspec_raw(obj, fp) - elif isinstance(obj, str): - _pack_string(obj, fp) - elif isinstance(obj, bytes): - _pack_binary(obj, fp) - elif isinstance(obj, list) or isinstance(obj, tuple): - _pack_array(obj, fp) - elif isinstance(obj, dict): - _pack_map(obj, fp) - elif isinstance(obj, Ext): - _pack_ext(obj, fp) - else: - raise UnsupportedTypeException("unsupported type: %s" % str(type(obj))) - -def _packb2(obj): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - - Returns: - A 'str' containing serialized MessagePack bytes. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> umsgpack.packb({u"compact": True, u"schema": 0}) - '\x82\xa7compact\xc3\xa6schema\x00' - >>> - """ - fp = io.BytesIO() - _pack2(obj, fp) - return fp.getvalue() - -def _packb3(obj): - """ - Serialize a Python object into MessagePack bytes. - - Args: - obj: a Python object - - Returns: - A 'bytes' containing serialized MessagePack bytes. - - Raises: - UnsupportedType(PackException): - Object type not supported for packing. - - Example: - >>> umsgpack.packb({u"compact": True, u"schema": 0}) - b'\x82\xa7compact\xc3\xa6schema\x00' - >>> - """ - fp = io.BytesIO() - _pack3(obj, fp) - return fp.getvalue() - -################################################################################ -### Unpacking -################################################################################ - -def _read_except(fp, n): - data = fp.read(n) - if len(data) < n: - raise InsufficientDataException() - return data - -def _unpack_integer(code, fp): - if (ord(code) & 0xe0) == 0xe0: - return struct.unpack("b", code)[0] - elif code == b'\xd0': - return struct.unpack("b", _read_except(fp, 1))[0] - elif code == b'\xd1': - return struct.unpack(">h", _read_except(fp, 2))[0] - elif code == b'\xd2': - return struct.unpack(">i", _read_except(fp, 4))[0] - elif code == b'\xd3': - return struct.unpack(">q", _read_except(fp, 8))[0] - elif (ord(code) & 0x80) == 0x00: - return struct.unpack("B", code)[0] - elif code == b'\xcc': - return struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xcd': - return struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xce': - return struct.unpack(">I", _read_except(fp, 4))[0] - elif code == b'\xcf': - return struct.unpack(">Q", _read_except(fp, 8))[0] - raise Exception("logic error, not int: 0x%02x" % ord(code)) - -def _unpack_reserved(code, fp): - if code == b'\xc1': - raise ReservedCodeException("encountered reserved code: 0x%02x" % ord(code)) - raise Exception("logic error, not reserved code: 0x%02x" % ord(code)) - -def _unpack_nil(code, fp): - if code == b'\xc0': - return None - raise Exception("logic error, not nil: 0x%02x" % ord(code)) - -def _unpack_boolean(code, fp): - if code == b'\xc2': - return False - elif code == b'\xc3': - return True - raise Exception("logic error, not boolean: 0x%02x" % ord(code)) - -def _unpack_float(code, fp): - if code == b'\xca': - return struct.unpack(">f", _read_except(fp, 4))[0] - elif code == b'\xcb': - return struct.unpack(">d", _read_except(fp, 8))[0] - raise Exception("logic error, not float: 0x%02x" % ord(code)) - -def _unpack_string(code, fp): - if (ord(code) & 0xe0) == 0xa0: - length = ord(code) & ~0xe0 - elif code == b'\xd9': - length = struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xda': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xdb': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not string: 0x%02x" % ord(code)) - - # Always return raw bytes in compatibility mode - global compatibility - if compatibility: - return _read_except(fp, length) - - try: - return bytes.decode(_read_except(fp, length), 'utf-8') - except UnicodeDecodeError: - raise InvalidStringException("unpacked string is not utf-8") - -def _unpack_binary(code, fp): - if code == b'\xc4': - length = struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xc5': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xc6': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not binary: 0x%02x" % ord(code)) - - return _read_except(fp, length) - -def _unpack_ext(code, fp): - if code == b'\xd4': - length = 1 - elif code == b'\xd5': - length = 2 - elif code == b'\xd6': - length = 4 - elif code == b'\xd7': - length = 8 - elif code == b'\xd8': - length = 16 - elif code == b'\xc7': - length = struct.unpack("B", _read_except(fp, 1))[0] - elif code == b'\xc8': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xc9': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not ext: 0x%02x" % ord(code)) - - return Ext(ord(_read_except(fp, 1)), _read_except(fp, length)) - -def _unpack_array(code, fp): - if (ord(code) & 0xf0) == 0x90: - length = (ord(code) & ~0xf0) - elif code == b'\xdc': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xdd': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not array: 0x%02x" % ord(code)) - - return [_unpack(fp) for i in xrange(length)] - -def _deep_list_to_tuple(obj): - if isinstance(obj, list): - return tuple([_deep_list_to_tuple(e) for e in obj]) - return obj - -def _unpack_map(code, fp): - if (ord(code) & 0xf0) == 0x80: - length = (ord(code) & ~0xf0) - elif code == b'\xde': - length = struct.unpack(">H", _read_except(fp, 2))[0] - elif code == b'\xdf': - length = struct.unpack(">I", _read_except(fp, 4))[0] - else: - raise Exception("logic error, not map: 0x%02x" % ord(code)) - - d = {} - for i in xrange(length): - # Unpack key - k = _unpack(fp) - - if isinstance(k, list): - # Attempt to convert list into a hashable tuple - k = _deep_list_to_tuple(k) - elif not isinstance(k, collections.Hashable): - raise UnhashableKeyException("encountered unhashable key: %s, %s" % (str(k), str(type(k)))) - elif k in d: - raise DuplicateKeyException("encountered duplicate key: %s, %s" % (str(k), str(type(k)))) - - # Unpack value - v = _unpack(fp) - - try: - d[k] = v - except TypeError: - raise UnhashableKeyException("encountered unhashable key: %s" % str(k)) - return d - -def _unpack(fp): - code = _read_except(fp, 1) - return _unpack_dispatch_table[code](code, fp) - -######################################## - -def _unpack2(fp): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - fp: a .read()-supporting file-like object - - Returns: - A Python object. - - Raises: - InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> f = open('test.bin', 'rb') - >>> umsgpack.unpackb(f) - {u'compact': True, u'schema': 0} - >>> - """ - return _unpack(fp) - -def _unpack3(fp): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - fp: a .read()-supporting file-like object - - Returns: - A Python object. - - Raises: - InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> f = open('test.bin', 'rb') - >>> umsgpack.unpackb(f) - {'compact': True, 'schema': 0} - >>> - """ - return _unpack(fp) - -# For Python 2, expects a str object -def _unpackb2(s): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - s: a 'str' containing serialized MessagePack bytes - - Returns: - A Python object. - - Raises: - TypeError: - Packed data is not type 'str'. - InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') - {u'compact': True, u'schema': 0} - >>> - """ - if not isinstance(s, str): - raise TypeError("packed data is not type 'str'") - return _unpack(io.BytesIO(s)) - -# For Python 3, expects a bytes object -def _unpackb3(s): - """ - Deserialize MessagePack bytes into a Python object. - - Args: - s: a 'bytes' containing serialized MessagePack bytes - - Returns: - A Python object. - - Raises: - TypeError: - Packed data is not type 'bytes'. - InsufficientDataException(UnpackException): - Insufficient data to unpack the encoded object. - InvalidStringException(UnpackException): - Invalid UTF-8 string encountered during unpacking. - ReservedCodeException(UnpackException): - Reserved code encountered during unpacking. - UnhashableKeyException(UnpackException): - Unhashable key encountered during map unpacking. - The serialized map cannot be deserialized into a Python dictionary. - DuplicateKeyException(UnpackException): - Duplicate key encountered during map unpacking. - - Example: - >>> umsgpack.unpackb(b'\x82\xa7compact\xc3\xa6schema\x00') - {'compact': True, 'schema': 0} - >>> - """ - if not isinstance(s, bytes): - raise TypeError("packed data is not type 'bytes'") - return _unpack(io.BytesIO(s)) - -################################################################################ -### Module Initialization -################################################################################ - -def __init(): - global pack - global packb - global unpack - global unpackb - global dump - global dumps - global load - global loads - global compatibility - global _float_size - global _unpack_dispatch_table - global xrange - - # Compatibility mode for handling strings/bytes with the old specification - compatibility = False - - # Auto-detect system float precision - if sys.float_info.mant_dig == 53: - _float_size = 64 - else: - _float_size = 32 - - # Map packb and unpackb to the appropriate version - if sys.version_info[0] == 3: - pack = _pack3 - packb = _packb3 - dump = _pack3 - dumps = _packb3 - unpack = _unpack3 - unpackb = _unpackb3 - load = _unpack3 - loads = _unpackb3 - xrange = range - else: - pack = _pack2 - packb = _packb2 - dump = _pack2 - dumps = _packb2 - unpack = _unpack2 - unpackb = _unpackb2 - load = _unpack2 - loads = _unpackb2 - - # Build a dispatch table for fast lookup of unpacking function - - _unpack_dispatch_table = {} - # Fix uint - for code in range(0, 0x7f+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - # Fix map - for code in range(0x80, 0x8f+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_map - # Fix array - for code in range(0x90, 0x9f+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_array - # Fix str - for code in range(0xa0, 0xbf+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string - # Nil - _unpack_dispatch_table[b'\xc0'] = _unpack_nil - # Reserved - _unpack_dispatch_table[b'\xc1'] = _unpack_reserved - # Boolean - _unpack_dispatch_table[b'\xc2'] = _unpack_boolean - _unpack_dispatch_table[b'\xc3'] = _unpack_boolean - # Bin - for code in range(0xc4, 0xc6+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_binary - # Ext - for code in range(0xc7, 0xc9+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext - # Float - _unpack_dispatch_table[b'\xca'] = _unpack_float - _unpack_dispatch_table[b'\xcb'] = _unpack_float - # Uint - for code in range(0xcc, 0xcf+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - # Int - for code in range(0xd0, 0xd3+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - # Fixext - for code in range(0xd4, 0xd8+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext - # String - for code in range(0xd9, 0xdb+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string - # Array - _unpack_dispatch_table[b'\xdc'] = _unpack_array - _unpack_dispatch_table[b'\xdd'] = _unpack_array - # Map - _unpack_dispatch_table[b'\xde'] = _unpack_map - _unpack_dispatch_table[b'\xdf'] = _unpack_map - # Negative fixint - for code in range(0xe0, 0xff+1): - _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer - -__init() diff --git a/umsgpack/__init__.py b/umsgpack/__init__.py new file mode 100644 index 0000000..e7e194f --- /dev/null +++ b/umsgpack/__init__.py @@ -0,0 +1,1248 @@ +# u-msgpack-python v2.8.0 - v at sergeev.io +# https://github.com/vsergeev/u-msgpack-python +# +# u-msgpack-python is a lightweight MessagePack serializer and deserializer +# module, compatible with both Python 2 and 3, as well CPython and PyPy +# implementations of Python. u-msgpack-python is fully compliant with the +# latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In +# particular, it supports the new binary, UTF-8 string, and application ext +# types. +# +# MIT License +# +# Copyright (c) 2013-2023 vsergeev / Ivan (Vanya) A. Sergeev +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +""" +u-msgpack-python v2.8.0 - v at sergeev.io +https://github.com/vsergeev/u-msgpack-python + +u-msgpack-python is a lightweight MessagePack serializer and deserializer +module, compatible with both Python 2 and 3, as well CPython and PyPy +implementations of Python. u-msgpack-python is fully compliant with the +latest MessagePack specification.com/msgpack/msgpack/blob/master/spec.md). In +particular, it supports the new binary, UTF-8 string, and application ext +types. + +License: MIT +""" +import struct +import collections +import datetime +import sys +import io + +if sys.version_info[0:2] >= (3, 3): + from collections.abc import Hashable +else: + from collections import Hashable + +__version__ = "2.8.0" +"Module version string" + +version = (2, 8, 0) +"Module version tuple" + + +############################################################################## +# Ext Class +############################################################################## + +# Extension type for application-defined types and data +class Ext(object): + """ + The Ext class facilitates creating a serializable extension object to store + an application-defined type and data byte array. + """ + + def __init__(self, type, data): + """ + Construct a new Ext object. + + Args: + type (int): application-defined type integer + data (bytes): application-defined data byte array + + Raises: + TypeError: + Type is not an integer. + ValueError: + Type is out of range of -128 to 127. + TypeError: + Data is not type 'bytes' (Python 3) or not type 'str' (Python 2). + + Example: + >>> foo = umsgpack.Ext(5, b"\\x01\\x02\\x03") + >>> umsgpack.packb({u"special stuff": foo, u"awesome": True}) + '\\x82\\xa7awesome\\xc3\\xadspecial stuff\\xc7\\x03\\x05\\x01\\x02\\x03' + >>> bar = umsgpack.unpackb(_) + >>> print(bar["special stuff"]) + Ext Object (Type: 5, Data: 01 02 03) + """ + # Check type is type int and in range + if not isinstance(type, int): + raise TypeError("ext type is not type integer") + elif not (-2**7 <= type <= 2**7 - 1): + raise ValueError("ext type value {:d} is out of range (-128 to 127)".format(type)) + # Check data is type bytes or str + elif sys.version_info[0] == 3 and not isinstance(data, bytes): + raise TypeError("ext data is not type \'bytes\'") + elif sys.version_info[0] == 2 and not isinstance(data, str): + raise TypeError("ext data is not type \'str\'") + + self.type = type + self.data = data + + def __eq__(self, other): + """ + Compare this Ext object with another for equality. + """ + return isinstance(other, self.__class__) \ + and self.type == other.type and self.data == other.data + + def __ne__(self, other): + """ + Compare this Ext object with another for inequality. + """ + return not self.__eq__(other) + + def __str__(self): + """ + String representation of this Ext object. + """ + s = "Ext Object (Type: {:d}, Data: ".format(self.type) + s += " ".join(["0x{:02x}".format(ord(self.data[i:i + 1])) + for i in xrange(min(len(self.data), 8))]) + if len(self.data) > 8: + s += " ..." + s += ")" + return s + + def __hash__(self): + """ + Provide a hash of this Ext object. + """ + return hash((self.type, self.data)) + + +class InvalidString(bytes): + """Subclass of bytes to hold invalid UTF-8 strings.""" + + +############################################################################## +# Ext Serializable Decorator +############################################################################## + +_ext_class_to_type = {} +_ext_type_to_class = {} + + +def ext_serializable(ext_type): + """ + Return a decorator to register a class for automatic packing and unpacking + with the specified Ext type code. The application class should implement a + `packb()` method that returns serialized bytes, and an `unpackb()` class + method or static method that accepts serialized bytes and returns an + instance of the application class. + + Args: + ext_type (int): application-defined Ext type code + + Raises: + TypeError: + Ext type is not an integer. + ValueError: + Ext type is out of range of -128 to 127. + ValueError: + Ext type or class already registered. + """ + def wrapper(cls): + if not isinstance(ext_type, int): + raise TypeError("Ext type is not type integer") + elif not (-2**7 <= ext_type <= 2**7 - 1): + raise ValueError("Ext type value {:d} is out of range of -128 to 127".format(ext_type)) + elif ext_type in _ext_type_to_class: + raise ValueError("Ext type {:d} already registered with class {:s}".format(ext_type, repr(_ext_type_to_class[ext_type]))) + elif cls in _ext_class_to_type: + raise ValueError("Class {:s} already registered with Ext type {:d}".format(repr(cls), ext_type)) + + _ext_type_to_class[ext_type] = cls + _ext_class_to_type[cls] = ext_type + + return cls + + return wrapper + + +############################################################################## +# Exceptions +############################################################################## + + +# Base Exception classes +class PackException(Exception): + "Base class for exceptions encountered during packing." + + +class UnpackException(Exception): + "Base class for exceptions encountered during unpacking." + + +# Packing error +class UnsupportedTypeException(PackException): + "Object type not supported for packing." + + +# Unpacking error +class InsufficientDataException(UnpackException): + "Insufficient data to unpack the serialized object." + + +class InvalidStringException(UnpackException): + "Invalid UTF-8 string encountered during unpacking." + + +class UnsupportedTimestampException(UnpackException): + "Unsupported timestamp format encountered during unpacking." + + +class ReservedCodeException(UnpackException): + "Reserved code encountered during unpacking." + + +class UnhashableKeyException(UnpackException): + """ + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + """ + + +class DuplicateKeyException(UnpackException): + "Duplicate key encountered during map unpacking." + + +# Backwards compatibility +KeyNotPrimitiveException = UnhashableKeyException +KeyDuplicateException = DuplicateKeyException + +############################################################################# +# Exported Functions and Glob +############################################################################# + +# Exported functions and variables, set up in __init() +pack = None +packb = None +unpack = None +unpackb = None +dump = None +dumps = None +load = None +loads = None + +compatibility = False +""" +Compatibility mode boolean. + +When compatibility mode is enabled, u-msgpack-python will serialize both +unicode strings and bytes into the old "raw" msgpack type, and deserialize the +"raw" msgpack type into bytes. This provides backwards compatibility with the +old MessagePack specification. + +Example: + >>> umsgpack.compatibility = True + >>> umsgpack.packb([u"some string", b"some bytes"]) + b'\\x92\\xabsome string\\xaasome bytes' + >>> umsgpack.unpackb(_) + [b'some string', b'some bytes'] +""" + +############################################################################## +# Packing +############################################################################## + +# You may notice struct.pack("B", obj) instead of the simpler chr(obj) in the +# code below. This is to allow for seamless Python 2 and 3 compatibility, as +# chr(obj) has a str return type instead of bytes in Python 3, and +# struct.pack(...) has the right return type in both versions. + + +def _pack_integer(obj, fp, options): + if obj < 0: + if obj >= -32: + fp.write(struct.pack("b", obj)) + elif obj >= -2**(8 - 1): + fp.write(b"\xd0" + struct.pack("b", obj)) + elif obj >= -2**(16 - 1): + fp.write(b"\xd1" + struct.pack(">h", obj)) + elif obj >= -2**(32 - 1): + fp.write(b"\xd2" + struct.pack(">i", obj)) + elif obj >= -2**(64 - 1): + fp.write(b"\xd3" + struct.pack(">q", obj)) + else: + raise UnsupportedTypeException("huge signed int") + else: + if obj < 128: + fp.write(struct.pack("B", obj)) + elif obj < 2**8: + fp.write(b"\xcc" + struct.pack("B", obj)) + elif obj < 2**16: + fp.write(b"\xcd" + struct.pack(">H", obj)) + elif obj < 2**32: + fp.write(b"\xce" + struct.pack(">I", obj)) + elif obj < 2**64: + fp.write(b"\xcf" + struct.pack(">Q", obj)) + else: + raise UnsupportedTypeException("huge unsigned int") + + +def _pack_nil(obj, fp, options): + fp.write(b"\xc0") + + +def _pack_boolean(obj, fp, options): + fp.write(b"\xc3" if obj else b"\xc2") + + +def _pack_float(obj, fp, options): + float_precision = options.get('force_float_precision', _float_precision) + + if float_precision == "double": + fp.write(b"\xcb" + struct.pack(">d", obj)) + elif float_precision == "single": + fp.write(b"\xca" + struct.pack(">f", obj)) + else: + raise ValueError("invalid float precision") + + +def _pack_string(obj, fp, options): + obj = obj.encode('utf-8') + obj_len = len(obj) + if obj_len < 32: + fp.write(struct.pack("B", 0xa0 | obj_len) + obj) + elif obj_len < 2**8: + fp.write(b"\xd9" + struct.pack("B", obj_len) + obj) + elif obj_len < 2**16: + fp.write(b"\xda" + struct.pack(">H", obj_len) + obj) + elif obj_len < 2**32: + fp.write(b"\xdb" + struct.pack(">I", obj_len) + obj) + else: + raise UnsupportedTypeException("huge string") + + +def _pack_binary(obj, fp, options): + obj_len = len(obj) + if obj_len < 2**8: + fp.write(b"\xc4" + struct.pack("B", obj_len) + obj) + elif obj_len < 2**16: + fp.write(b"\xc5" + struct.pack(">H", obj_len) + obj) + elif obj_len < 2**32: + fp.write(b"\xc6" + struct.pack(">I", obj_len) + obj) + else: + raise UnsupportedTypeException("huge binary string") + + +def _pack_oldspec_raw(obj, fp, options): + obj_len = len(obj) + if obj_len < 32: + fp.write(struct.pack("B", 0xa0 | obj_len) + obj) + elif obj_len < 2**16: + fp.write(b"\xda" + struct.pack(">H", obj_len) + obj) + elif obj_len < 2**32: + fp.write(b"\xdb" + struct.pack(">I", obj_len) + obj) + else: + raise UnsupportedTypeException("huge raw string") + + +def _pack_ext(obj, fp, options): + obj_len = len(obj.data) + if obj_len == 1: + fp.write(b"\xd4" + struct.pack("B", obj.type & 0xff) + obj.data) + elif obj_len == 2: + fp.write(b"\xd5" + struct.pack("B", obj.type & 0xff) + obj.data) + elif obj_len == 4: + fp.write(b"\xd6" + struct.pack("B", obj.type & 0xff) + obj.data) + elif obj_len == 8: + fp.write(b"\xd7" + struct.pack("B", obj.type & 0xff) + obj.data) + elif obj_len == 16: + fp.write(b"\xd8" + struct.pack("B", obj.type & 0xff) + obj.data) + elif obj_len < 2**8: + fp.write(b"\xc7" + struct.pack("BB", obj_len, obj.type & 0xff) + obj.data) + elif obj_len < 2**16: + fp.write(b"\xc8" + struct.pack(">HB", obj_len, obj.type & 0xff) + obj.data) + elif obj_len < 2**32: + fp.write(b"\xc9" + struct.pack(">IB", obj_len, obj.type & 0xff) + obj.data) + else: + raise UnsupportedTypeException("huge ext data") + + +def _pack_ext_timestamp(obj, fp, options): + if not obj.tzinfo: + # Object is naive datetime, convert to aware date time, + # assuming UTC timezone + delta = obj.replace(tzinfo=_utc_tzinfo) - _epoch + else: + # Object is aware datetime + delta = obj - _epoch + + seconds = delta.seconds + delta.days * 86400 + microseconds = delta.microseconds + + if microseconds == 0 and 0 <= seconds <= 2**32 - 1: + # 32-bit timestamp + fp.write(b"\xd6\xff" + struct.pack(">I", seconds)) + elif 0 <= seconds <= 2**34 - 1: + # 64-bit timestamp + value = ((microseconds * 1000) << 34) | seconds + fp.write(b"\xd7\xff" + struct.pack(">Q", value)) + elif -2**63 <= abs(seconds) <= 2**63 - 1: + # 96-bit timestamp + fp.write(b"\xc7\x0c\xff" + struct.pack(">Iq", microseconds * 1000, seconds)) + else: + raise UnsupportedTypeException("huge timestamp") + + +def _pack_array(obj, fp, options): + obj_len = len(obj) + if obj_len < 16: + fp.write(struct.pack("B", 0x90 | obj_len)) + elif obj_len < 2**16: + fp.write(b"\xdc" + struct.pack(">H", obj_len)) + elif obj_len < 2**32: + fp.write(b"\xdd" + struct.pack(">I", obj_len)) + else: + raise UnsupportedTypeException("huge array") + + for e in obj: + pack(e, fp, **options) + + +def _pack_map(obj, fp, options): + obj_len = len(obj) + if obj_len < 16: + fp.write(struct.pack("B", 0x80 | obj_len)) + elif obj_len < 2**16: + fp.write(b"\xde" + struct.pack(">H", obj_len)) + elif obj_len < 2**32: + fp.write(b"\xdf" + struct.pack(">I", obj_len)) + else: + raise UnsupportedTypeException("huge array") + + for k, v in obj.items(): + pack(k, fp, **options) + pack(v, fp, **options) + +######################################## + + +# Pack for Python 2, with 'unicode' type, 'str' type, and 'long' type +def _pack2(obj, fp, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + fp: a .write()-supporting file-like object + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats + + Returns: + None + + Raises: + UnsupportedTypeException(PackException): + Object type not supported for packing. + + Example: + >>> f = open('test.bin', 'wb') + >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) + """ + global compatibility + + ext_handlers = options.get("ext_handlers") + + if obj is None: + _pack_nil(obj, fp, options) + elif ext_handlers and obj.__class__ in ext_handlers: + _pack_ext(ext_handlers[obj.__class__](obj), fp, options) + elif obj.__class__ in _ext_class_to_type: + try: + _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) + elif isinstance(obj, bool): + _pack_boolean(obj, fp, options) + elif isinstance(obj, (int, long)): # noqa: F821 + _pack_integer(obj, fp, options) + elif isinstance(obj, float): + _pack_float(obj, fp, options) + elif compatibility and isinstance(obj, unicode): # noqa: F821 + _pack_oldspec_raw(bytes(obj), fp, options) + elif compatibility and isinstance(obj, bytes): + _pack_oldspec_raw(obj, fp, options) + elif isinstance(obj, unicode): # noqa: F821 + _pack_string(obj, fp, options) + elif isinstance(obj, str): + _pack_binary(obj, fp, options) + elif isinstance(obj, (list, tuple)): + _pack_array(obj, fp, options) + elif isinstance(obj, dict): + _pack_map(obj, fp, options) + elif isinstance(obj, datetime.datetime): + _pack_ext_timestamp(obj, fp, options) + elif isinstance(obj, Ext): + _pack_ext(obj, fp, options) + elif ext_handlers: + # Linear search for superclass + t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) + if t: + _pack_ext(ext_handlers[t](obj), fp, options) + else: + raise UnsupportedTypeException( + "unsupported type: {:s}".format(str(type(obj)))) + elif _ext_class_to_type: + # Linear search for superclass + t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) + if t: + try: + _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) + else: + raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj)))) + else: + raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj)))) + + +# Pack for Python 3, with unicode 'str' type, 'bytes' type, and no 'long' type +def _pack3(obj, fp, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + fp: a .write()-supporting file-like object + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats + + Returns: + None + + Raises: + UnsupportedTypeException(PackException): + Object type not supported for packing. + + Example: + >>> f = open('test.bin', 'wb') + >>> umsgpack.pack({u"compact": True, u"schema": 0}, f) + """ + global compatibility + + ext_handlers = options.get("ext_handlers") + + if obj is None: + _pack_nil(obj, fp, options) + elif ext_handlers and obj.__class__ in ext_handlers: + _pack_ext(ext_handlers[obj.__class__](obj), fp, options) + elif obj.__class__ in _ext_class_to_type: + try: + _pack_ext(Ext(_ext_class_to_type[obj.__class__], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(obj.__class__))) + elif isinstance(obj, bool): + _pack_boolean(obj, fp, options) + elif isinstance(obj, int): + _pack_integer(obj, fp, options) + elif isinstance(obj, float): + _pack_float(obj, fp, options) + elif compatibility and isinstance(obj, str): + _pack_oldspec_raw(obj.encode('utf-8'), fp, options) + elif compatibility and isinstance(obj, bytes): + _pack_oldspec_raw(obj, fp, options) + elif isinstance(obj, str): + _pack_string(obj, fp, options) + elif isinstance(obj, bytes): + _pack_binary(obj, fp, options) + elif isinstance(obj, (list, tuple)): + _pack_array(obj, fp, options) + elif isinstance(obj, dict): + _pack_map(obj, fp, options) + elif isinstance(obj, datetime.datetime): + _pack_ext_timestamp(obj, fp, options) + elif isinstance(obj, Ext): + _pack_ext(obj, fp, options) + elif ext_handlers: + # Linear search for superclass + t = next((t for t in ext_handlers.keys() if isinstance(obj, t)), None) + if t: + _pack_ext(ext_handlers[t](obj), fp, options) + else: + raise UnsupportedTypeException( + "unsupported type: {:s}".format(str(type(obj)))) + elif _ext_class_to_type: + # Linear search for superclass + t = next((t for t in _ext_class_to_type if isinstance(obj, t)), None) + if t: + try: + _pack_ext(Ext(_ext_class_to_type[t], obj.packb()), fp, options) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of packb()".format(repr(t))) + else: + raise UnsupportedTypeException("unsupported type: {:s}".format(str(type(obj)))) + else: + raise UnsupportedTypeException( + "unsupported type: {:s}".format(str(type(obj)))) + + +def _packb2(obj, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats + + Returns: + str: Serialized MessagePack bytes + + Raises: + UnsupportedTypeException(PackException): + Object type not supported for packing. + + Example: + >>> umsgpack.packb({u"compact": True, u"schema": 0}) + '\\x82\\xa7compact\\xc3\\xa6schema\\x00' + """ + fp = io.BytesIO() + _pack2(obj, fp, **options) + return fp.getvalue() + + +def _packb3(obj, **options): + """ + Serialize a Python object into MessagePack bytes. + + Args: + obj: a Python object + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping a custom type + to a callable that packs an instance of the type + into an Ext object + force_float_precision (str): "single" to force packing floats as + IEEE-754 single-precision floats, + "double" to force packing floats as + IEEE-754 double-precision floats + + Returns: + bytes: Serialized MessagePack bytes + + Raises: + UnsupportedTypeException(PackException): + Object type not supported for packing. + + Example: + >>> umsgpack.packb({u"compact": True, u"schema": 0}) + b'\\x82\\xa7compact\\xc3\\xa6schema\\x00' + """ + fp = io.BytesIO() + _pack3(obj, fp, **options) + return fp.getvalue() + +############################################################################# +# Unpacking +############################################################################# + + +def _read_except(fp, n): + if n == 0: + return b"" + + data = fp.read(n) + if len(data) == 0: + raise InsufficientDataException() + + while len(data) < n: + chunk = fp.read(n - len(data)) + if len(chunk) == 0: + raise InsufficientDataException() + + data += chunk + + return data + + +def _unpack_integer(code, fp, options): + if (ord(code) & 0xe0) == 0xe0: + return struct.unpack("b", code)[0] + elif code == b'\xd0': + return struct.unpack("b", _read_except(fp, 1))[0] + elif code == b'\xd1': + return struct.unpack(">h", _read_except(fp, 2))[0] + elif code == b'\xd2': + return struct.unpack(">i", _read_except(fp, 4))[0] + elif code == b'\xd3': + return struct.unpack(">q", _read_except(fp, 8))[0] + elif (ord(code) & 0x80) == 0x00: + return struct.unpack("B", code)[0] + elif code == b'\xcc': + return struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xcd': + return struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xce': + return struct.unpack(">I", _read_except(fp, 4))[0] + elif code == b'\xcf': + return struct.unpack(">Q", _read_except(fp, 8))[0] + raise Exception("logic error, not int: 0x{:02x}".format(ord(code))) + + +def _unpack_reserved(code, fp, options): + if code == b'\xc1': + raise ReservedCodeException( + "encountered reserved code: 0x{:02x}".format(ord(code))) + raise Exception( + "logic error, not reserved code: 0x{:02x}".format(ord(code))) + + +def _unpack_nil(code, fp, options): + if code == b'\xc0': + return None + raise Exception("logic error, not nil: 0x{:02x}".format(ord(code))) + + +def _unpack_boolean(code, fp, options): + if code == b'\xc2': + return False + elif code == b'\xc3': + return True + raise Exception("logic error, not boolean: 0x{:02x}".format(ord(code))) + + +def _unpack_float(code, fp, options): + if code == b'\xca': + return struct.unpack(">f", _read_except(fp, 4))[0] + elif code == b'\xcb': + return struct.unpack(">d", _read_except(fp, 8))[0] + raise Exception("logic error, not float: 0x{:02x}".format(ord(code))) + + +def _unpack_string(code, fp, options): + if (ord(code) & 0xe0) == 0xa0: + length = ord(code) & ~0xe0 + elif code == b'\xd9': + length = struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xda': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xdb': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not string: 0x{:02x}".format(ord(code))) + + # Always return raw bytes in compatibility mode + global compatibility + if compatibility: + return _read_except(fp, length) + + data = _read_except(fp, length) + try: + return bytes.decode(data, 'utf-8') + except UnicodeDecodeError: + if options.get("allow_invalid_utf8"): + return InvalidString(data) + raise InvalidStringException("unpacked string is invalid utf-8") + + +def _unpack_binary(code, fp, options): + if code == b'\xc4': + length = struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xc5': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xc6': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not binary: 0x{:02x}".format(ord(code))) + + return _read_except(fp, length) + + +def _unpack_ext(code, fp, options): + if code == b'\xd4': + length = 1 + elif code == b'\xd5': + length = 2 + elif code == b'\xd6': + length = 4 + elif code == b'\xd7': + length = 8 + elif code == b'\xd8': + length = 16 + elif code == b'\xc7': + length = struct.unpack("B", _read_except(fp, 1))[0] + elif code == b'\xc8': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xc9': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not ext: 0x{:02x}".format(ord(code))) + + ext_type = struct.unpack("b", _read_except(fp, 1))[0] + ext_data = _read_except(fp, length) + + # Unpack with ext handler, if we have one + ext_handlers = options.get("ext_handlers") + if ext_handlers and ext_type in ext_handlers: + return ext_handlers[ext_type](Ext(ext_type, ext_data)) + + # Unpack with ext classes, if type is registered + if ext_type in _ext_type_to_class: + try: + return _ext_type_to_class[ext_type].unpackb(ext_data) + except AttributeError: + raise NotImplementedError("Ext serializable class {:s} is missing implementation of unpackb()".format(repr(_ext_type_to_class[ext_type]))) + + # Timestamp extension + if ext_type == -1: + return _unpack_ext_timestamp(ext_data, options) + + return Ext(ext_type, ext_data) + + +def _unpack_ext_timestamp(ext_data, options): + obj_len = len(ext_data) + if obj_len == 4: + # 32-bit timestamp + seconds = struct.unpack(">I", ext_data)[0] + microseconds = 0 + elif obj_len == 8: + # 64-bit timestamp + value = struct.unpack(">Q", ext_data)[0] + seconds = value & 0x3ffffffff + microseconds = (value >> 34) // 1000 + elif obj_len == 12: + # 96-bit timestamp + seconds = struct.unpack(">q", ext_data[4:12])[0] + microseconds = struct.unpack(">I", ext_data[0:4])[0] // 1000 + else: + raise UnsupportedTimestampException( + "unsupported timestamp with data length {:d}".format(len(ext_data))) + + return _epoch + datetime.timedelta(seconds=seconds, + microseconds=microseconds) + + +def _unpack_array(code, fp, options): + if (ord(code) & 0xf0) == 0x90: + length = (ord(code) & ~0xf0) + elif code == b'\xdc': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xdd': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not array: 0x{:02x}".format(ord(code))) + + if options.get('use_tuple'): + return tuple((_unpack(fp, options) for i in xrange(length))) + + return [_unpack(fp, options) for i in xrange(length)] + + +def _deep_list_to_tuple(obj): + if isinstance(obj, list): + return tuple([_deep_list_to_tuple(e) for e in obj]) + return obj + + +def _unpack_map(code, fp, options): + if (ord(code) & 0xf0) == 0x80: + length = (ord(code) & ~0xf0) + elif code == b'\xde': + length = struct.unpack(">H", _read_except(fp, 2))[0] + elif code == b'\xdf': + length = struct.unpack(">I", _read_except(fp, 4))[0] + else: + raise Exception("logic error, not map: 0x{:02x}".format(ord(code))) + + d = {} if not options.get('use_ordered_dict') else collections.OrderedDict() + for _ in xrange(length): + # Unpack key + k = _unpack(fp, options) + + if isinstance(k, list): + # Attempt to convert list into a hashable tuple + k = _deep_list_to_tuple(k) + elif not isinstance(k, Hashable): + raise UnhashableKeyException( + "encountered unhashable key: \"{:s}\" ({:s})".format(str(k), str(type(k)))) + elif k in d: + raise DuplicateKeyException( + "encountered duplicate key: \"{:s}\" ({:s})".format(str(k), str(type(k)))) + + # Unpack value + v = _unpack(fp, options) + + try: + d[k] = v + except TypeError: + raise UnhashableKeyException( + "encountered unhashable key: \"{:s}\"".format(str(k))) + return d + + +def _unpack(fp, options): + code = _read_except(fp, 1) + return _unpack_dispatch_table[code](code, fp, options) + +######################################## + + +def _unpack2(fp, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + fp: a .read()-supporting file-like object + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + :class:`InvalidString`, for access to the + bytes (default False) + + Returns: + Python object + + Raises: + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> f = open('test.bin', 'rb') + >>> umsgpack.unpackb(f) + {u'compact': True, u'schema': 0} + """ + return _unpack(fp, options) + + +def _unpack3(fp, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + fp: a .read()-supporting file-like object + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + :class:`InvalidString`, for access to the + bytes (default False) + + Returns: + Python object + + Raises: + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> f = open('test.bin', 'rb') + >>> umsgpack.unpackb(f) + {'compact': True, 'schema': 0} + """ + return _unpack(fp, options) + + +# For Python 2, expects a str object +def _unpackb2(s, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + s (str, bytearray): serialized MessagePack bytes + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + :class:`InvalidString`, for access to the + bytes (default False) + + Returns: + Python object + + Raises: + TypeError: + Packed data type is neither 'str' nor 'bytearray'. + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> umsgpack.unpackb(b'\\x82\\xa7compact\\xc3\\xa6schema\\x00') + {u'compact': True, u'schema': 0} + """ + if not isinstance(s, (str, bytearray)): + raise TypeError("packed data must be type 'str' or 'bytearray'") + return _unpack(io.BytesIO(s), options) + + +# For Python 3, expects a bytes object +def _unpackb3(s, **options): + """ + Deserialize MessagePack bytes into a Python object. + + Args: + s (bytes, bytearray): serialized MessagePack bytes + + Keyword Args: + ext_handlers (dict): dictionary of Ext handlers, mapping integer Ext + type to a callable that unpacks an instance of + Ext into an object + use_ordered_dict (bool): unpack maps into OrderedDict, instead of dict + (default False) + use_tuple (bool): unpacks arrays into tuples, instead of lists (default + False) + allow_invalid_utf8 (bool): unpack invalid strings into instances of + :class:`InvalidString`, for access to the + bytes (default False) + + Returns: + Python object + + Raises: + TypeError: + Packed data type is neither 'bytes' nor 'bytearray'. + InsufficientDataException(UnpackException): + Insufficient data to unpack the serialized object. + InvalidStringException(UnpackException): + Invalid UTF-8 string encountered during unpacking. + UnsupportedTimestampException(UnpackException): + Unsupported timestamp format encountered during unpacking. + ReservedCodeException(UnpackException): + Reserved code encountered during unpacking. + UnhashableKeyException(UnpackException): + Unhashable key encountered during map unpacking. + The serialized map cannot be deserialized into a Python dictionary. + DuplicateKeyException(UnpackException): + Duplicate key encountered during map unpacking. + + Example: + >>> umsgpack.unpackb(b'\\x82\\xa7compact\\xc3\\xa6schema\\x00') + {'compact': True, 'schema': 0} + """ + if not isinstance(s, (bytes, bytearray)): + raise TypeError("packed data must be type 'bytes' or 'bytearray'") + return _unpack(io.BytesIO(s), options) + +############################################################################# +# Module Initialization +############################################################################# + + +def __init(): + global pack + global packb + global unpack + global unpackb + global dump + global dumps + global load + global loads + global compatibility + global _epoch + global _utc_tzinfo + global _float_precision + global _unpack_dispatch_table + global xrange + + # Compatibility mode for handling strings/bytes with the old specification + compatibility = False + + if sys.version_info[0] == 3: + _utc_tzinfo = datetime.timezone.utc + else: + class UTC(datetime.tzinfo): + ZERO = datetime.timedelta(0) + + def utcoffset(self, dt): + return UTC.ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return UTC.ZERO + + _utc_tzinfo = UTC() + + # Calculate an aware epoch datetime + _epoch = datetime.datetime(1970, 1, 1, tzinfo=_utc_tzinfo) + + # Auto-detect system float precision + if sys.float_info.mant_dig == 53: + _float_precision = "double" + else: + _float_precision = "single" + + # Map packb and unpackb to the appropriate version + if sys.version_info[0] == 3: + pack = _pack3 + packb = _packb3 + dump = _pack3 + dumps = _packb3 + unpack = _unpack3 + unpackb = _unpackb3 + load = _unpack3 + loads = _unpackb3 + xrange = range + else: + pack = _pack2 + packb = _packb2 + dump = _pack2 + dumps = _packb2 + unpack = _unpack2 + unpackb = _unpackb2 + load = _unpack2 + loads = _unpackb2 + + # Build a dispatch table for fast lookup of unpacking function + + _unpack_dispatch_table = {} + # Fix uint + for code in range(0, 0x7f + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + # Fix map + for code in range(0x80, 0x8f + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_map + # Fix array + for code in range(0x90, 0x9f + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_array + # Fix str + for code in range(0xa0, 0xbf + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string + # Nil + _unpack_dispatch_table[b'\xc0'] = _unpack_nil + # Reserved + _unpack_dispatch_table[b'\xc1'] = _unpack_reserved + # Boolean + _unpack_dispatch_table[b'\xc2'] = _unpack_boolean + _unpack_dispatch_table[b'\xc3'] = _unpack_boolean + # Bin + for code in range(0xc4, 0xc6 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_binary + # Ext + for code in range(0xc7, 0xc9 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext + # Float + _unpack_dispatch_table[b'\xca'] = _unpack_float + _unpack_dispatch_table[b'\xcb'] = _unpack_float + # Uint + for code in range(0xcc, 0xcf + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + # Int + for code in range(0xd0, 0xd3 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + # Fixext + for code in range(0xd4, 0xd8 + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_ext + # String + for code in range(0xd9, 0xdb + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_string + # Array + _unpack_dispatch_table[b'\xdc'] = _unpack_array + _unpack_dispatch_table[b'\xdd'] = _unpack_array + # Map + _unpack_dispatch_table[b'\xde'] = _unpack_map + _unpack_dispatch_table[b'\xdf'] = _unpack_map + # Negative fixint + for code in range(0xe0, 0xff + 1): + _unpack_dispatch_table[struct.pack("B", code)] = _unpack_integer + + +__init() diff --git a/umsgpack/__init__.pyi b/umsgpack/__init__.pyi new file mode 100644 index 0000000..61a8413 --- /dev/null +++ b/umsgpack/__init__.pyi @@ -0,0 +1,41 @@ +from typing import Any + +__version__: str + +version: tuple[int, int, int] + +def pack(obj, fp, **options) -> None: ... +def packb(obj, **options) -> bytes: ... +def dump(obj, fp, **options) -> None: ... +def dumps(obj, **options) -> bytes: ... + +def unpackb(s: bytes | bytearray, **options) -> Any: ... +def unpack(fp, **options) -> Any: ... +def loads(s: bytes | bytearray, **options) -> Any: ... +def load(fp, **options) -> Any: ... + +class Ext: + type: int + data: bytes + def __init__(self, type: int, data: bytes) -> None: ... + def __eq__(self, other) -> bool: ... + def __ne__(self, other) -> bool: ... + def __hash__(self) -> int: ... + +class InvalidString(bytes): ... + +def ext_serializable(ext_type: int): ... + +class PackException(Exception): ... +class UnpackException(Exception): ... +class UnsupportedTypeException(PackException): ... +class InsufficientDataException(UnpackException): ... +class InvalidStringException(UnpackException): ... +class UnsupportedTimestampException(UnpackException): ... +class ReservedCodeException(UnpackException): ... +class UnhashableKeyException(UnpackException): ... +class DuplicateKeyException(UnpackException): ... +KeyNotPrimitiveException = UnhashableKeyException +KeyDuplicateException = DuplicateKeyException + +compatibility: bool