From ea90564e5f5276a37df06f7d3dadf90faa210d7f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:13:27 +0100 Subject: [PATCH 0001/1043] Bump pypa/gh-action-pypi-publish from 1.8.10 to 1.8.11 (#1586) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.10 to 1.8.11. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.10...v1.8.11) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index c08bfc6677..3bd25bfbf7 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.10 + - uses: pypa/gh-action-pypi-publish@v1.8.11 with: user: __token__ password: ${{ secrets.pypi_password }} From 79e80b36b14c50c6d522f0fe0caaee0bbfbce1a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 09:13:52 +0100 Subject: [PATCH 0002/1043] Bump conda-incubator/setup-miniconda from 2.3.0 to 3.0.1 (#1587) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2.3.0 to 3.0.1. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v2.3.0...v3.0.1) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index 2c0cd45ca9..2cc0213781 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.3.0 + uses: conda-incubator/setup-miniconda@v3.0.1 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index aa7158f1cf..0c3c49d78d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,7 +42,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v2.3.0 + uses: conda-incubator/setup-miniconda@v3.0.1 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 78945e97aa..eeee5b704d 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v2.3.0 + - uses: conda-incubator/setup-miniconda@v3.0.1 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From 25dbeeda7d3a300569b358c157c5bd1c02ddaec3 Mon Sep 17 00:00:00 2001 From: Janick Martinez Esturo Date: Tue, 5 Dec 2023 22:31:03 +0100 Subject: [PATCH 0003/1043] * Cache result of FSStore._fsspec_installed (#1581) Prevent runtime-overhead in doing this check multiple times --- docs/release.rst | 3 +++ zarr/storage.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 9873d62896..842c36e290 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -43,6 +43,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Cache result of ``FSStore._fsspec_installed()``. + By :user:`Janick Martinez Esturo ` :issue:`1581`. + * Extend copyright notice to 2023. By :user:`Jack Kelly ` :issue:`1528`. diff --git a/zarr/storage.py b/zarr/storage.py index b36f804ebd..a7426e5345 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -28,6 +28,7 @@ import zipfile from collections import OrderedDict from collections.abc import MutableMapping +from functools import lru_cache from os import scandir from pickle import PicklingError from threading import Lock, RLock @@ -1540,6 +1541,7 @@ def clear(self): self.map.clear() @classmethod + @lru_cache(maxsize=None) def _fsspec_installed(cls): """Returns true if fsspec is installed""" import importlib.util From 8579e21c80927afbc26153153ca8eedc91a6ff6f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 7 Dec 2023 21:15:55 +0000 Subject: [PATCH 0004/1043] Bump version of black in pre-commit (#1559) --- .pre-commit-config.yaml | 2 +- bench/compress_normal.py | 1 - zarr/_storage/absstore.py | 3 +- zarr/_storage/store.py | 1 - zarr/_storage/v3.py | 1 - zarr/attrs.py | 6 ---- zarr/convenience.py | 20 +++-------- zarr/creation.py | 2 -- zarr/hierarchy.py | 12 +++---- zarr/indexing.py | 25 -------------- zarr/meta.py | 1 - zarr/n5.py | 57 -------------------------------- zarr/storage.py | 5 --- zarr/tests/test_attrs.py | 6 ---- zarr/tests/test_convenience.py | 7 ---- zarr/tests/test_creation.py | 9 ----- zarr/tests/test_dim_separator.py | 1 - zarr/tests/test_filters.py | 12 ------- zarr/tests/test_hierarchy.py | 3 -- zarr/tests/test_indexing.py | 35 -------------------- zarr/tests/test_info.py | 1 - zarr/tests/test_meta.py | 19 ----------- zarr/tests/test_storage.py | 20 ----------- zarr/tests/test_storage_v3.py | 10 ------ zarr/tests/test_sync.py | 2 -- zarr/tests/test_util.py | 2 -- zarr/util.py | 6 ---- 27 files changed, 11 insertions(+), 258 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f22dc39832..e985d24000 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ repos: # Respect `exclude` and `extend-exclude` settings. args: ["--force-exclude"] - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.10.1 hooks: - id: black - repo: https://github.com/codespell-project/codespell diff --git a/bench/compress_normal.py b/bench/compress_normal.py index 9f1655541c..803d54b76b 100644 --- a/bench/compress_normal.py +++ b/bench/compress_normal.py @@ -8,7 +8,6 @@ from zarr import blosc if __name__ == "__main__": - sys.path.insert(0, "..") # setup diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index f62529f096..c9a113148c 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -87,7 +87,7 @@ def __init__( "https://{}.blob.core.windows.net/".format(account_name), container, credential=account_key, - **blob_service_kwargs + **blob_service_kwargs, ) self.client = client @@ -240,7 +240,6 @@ def __setitem__(self, key, value): super().__setitem__(key, value) def rmdir(self, path=None): - if not path: # Currently allowing clear to delete everything as in v2 diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 8daedae48f..80e4ad8f75 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -629,7 +629,6 @@ def _rmdir_from_keys(store: StoreLike, path: Optional[str] = None) -> None: def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None: - meta_dir = meta_root + path meta_dir = meta_dir.rstrip("/") _rmdir_from_keys(store, meta_dir) diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 00dc085dac..32e78f7a34 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -118,7 +118,6 @@ def _get_files_and_dirs_from_path(store, path): class FSStoreV3(FSStore, StoreV3): - # FSStoreV3 doesn't use this (FSStore uses it within _normalize_key) _META_KEYS = () diff --git a/zarr/attrs.py b/zarr/attrs.py index 01fc617b3c..e967c5b853 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -26,7 +26,6 @@ class Attributes(MutableMapping): """ def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None): - self._version = getattr(store, "_store_version", 2) _Store = Store if self._version == 2 else StoreV3 self.store = _Store._ensure_store(store) @@ -73,7 +72,6 @@ def __getitem__(self, item): return self.asdict()[item] def _write_op(self, f, *args, **kwargs): - # guard condition if self.read_only: raise PermissionError("attributes are read-only") @@ -89,7 +87,6 @@ def __setitem__(self, item, value): self._write_op(self._setitem_nosync, item, value) def _setitem_nosync(self, item, value): - # load existing data d = self._get_nosync() @@ -106,7 +103,6 @@ def __delitem__(self, item): self._write_op(self._delitem_nosync, item) def _delitem_nosync(self, key): - # load existing data d = self._get_nosync() @@ -128,7 +124,6 @@ def put(self, d): self._write_op(self._put_nosync, dict(attributes=d)) def _put_nosync(self, d): - d_to_check = d if self._version == 2 else d["attributes"] if not all(isinstance(item, str) for item in d_to_check): # TODO: Raise an error for non-string keys @@ -178,7 +173,6 @@ def update(self, *args, **kwargs): self._write_op(self._update_nosync, *args, **kwargs) def _update_nosync(self, *args, **kwargs): - # load existing data d = self._get_nosync() diff --git a/zarr/convenience.py b/zarr/convenience.py index 0ee8a8d323..9c0deeea47 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -675,10 +675,8 @@ def copy_store( # setup logging with _LogWriter(log) as log: - # iterate over source keys for source_key in sorted(source.keys()): - # filter to keys under source path if source_store_version == 2: if not source_key.startswith(source_path): @@ -757,7 +755,7 @@ def copy( log=None, if_exists="raise", dry_run=False, - **create_kws + **create_kws, ): """Copy the `source` array or group into the `dest` group. @@ -878,7 +876,6 @@ def copy( # setup logging with _LogWriter(log) as log: - # do the copying n_copied, n_skipped, n_bytes_copied = _copy( log, @@ -890,7 +887,7 @@ def copy( without_attrs=without_attrs, if_exists=if_exists, dry_run=dry_run, - **create_kws + **create_kws, ) # log a final message with a summary of what happened @@ -948,12 +945,10 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # take action if do_copy: - # log a message about what we're going to do log("copy {} {} {}".format(source.name, source.shape, source.dtype)) if not dry_run: - # clear the way if exists: del dest[name] @@ -1038,12 +1033,10 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # take action if do_copy: - # log action log("copy {}".format(source.name)) if not dry_run: - # clear the way if exists_array: del dest[name] @@ -1056,7 +1049,6 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ grp.attrs.update(source.attrs) else: - # setup for dry run without creating any groups in the # destination if dest is not None: @@ -1076,7 +1068,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ without_attrs=without_attrs, if_exists=if_exists, dry_run=dry_run, - **create_kws + **create_kws, ) n_copied += c n_skipped += s @@ -1099,7 +1091,7 @@ def copy_all( log=None, if_exists="raise", dry_run=False, - **create_kws + **create_kws, ): """Copy all children of the `source` group into the `dest` group. @@ -1189,7 +1181,6 @@ def copy_all( # setup logging with _LogWriter(log) as log: - for k in source.keys(): c, s, b = _copy( log, @@ -1201,7 +1192,7 @@ def copy_all( without_attrs=without_attrs, if_exists=if_exists, dry_run=dry_run, - **create_kws + **create_kws, ) n_copied += c n_skipped += s @@ -1262,7 +1253,6 @@ def is_zarr_key(key): return key.endswith(".zarray") or key.endswith(".zgroup") or key.endswith(".zattrs") else: - assert_zarr_v3_api_available() sfx = _get_metadata_suffix(store) # type: ignore diff --git a/zarr/creation.py b/zarr/creation.py index 726d0b5932..6227f90b7b 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -234,7 +234,6 @@ def create( def _kwargs_compat(compressor, fill_value, kwargs): - # to be compatible with h5py, as well as backwards-compatible with Zarr # 1.x, accept 'compression' and 'compression_opts' keyword arguments @@ -697,7 +696,6 @@ def open_array( def _like_args(a, kwargs): - shape, chunks = _get_shape_chunks(a) if shape is not None: kwargs.setdefault("shape", shape) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 3361969f08..1cfea89c81 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -145,7 +145,7 @@ def __init__( synchronizer=None, zarr_version=None, *, - meta_array=None + meta_array=None, ): store: BaseStore = _normalize_store_arg(store, zarr_version=zarr_version) if zarr_version is None: @@ -919,7 +919,6 @@ def tree(self, expand=False, level=None): return TreeViewer(self, expand=expand, level=level) def _write_op(self, f, *args, **kwargs): - # guard condition if self._read_only: raise ReadOnlyError() @@ -1094,7 +1093,6 @@ def create_dataset(self, name, **kwargs): return self._write_op(self._create_dataset_nosync, name, **kwargs) def _create_dataset_nosync(self, name, data=None, **kwargs): - assert "mode" not in kwargs path = self._item_path(name) @@ -1138,11 +1136,9 @@ def require_dataset(self, name, shape, dtype=None, exact=False, **kwargs): ) def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, **kwargs): - path = self._item_path(name) if contains_array(self._store, path): - # array already exists at path, validate that it is the right shape and type synchronizer = kwargs.get("synchronizer", self._synchronizer) @@ -1235,7 +1231,7 @@ def _full_nosync(self, name, fill_value, **kwargs): path=path, chunk_store=self._chunk_store, fill_value=fill_value, - **kwargs + **kwargs, ) def array(self, name, data, **kwargs): @@ -1361,7 +1357,7 @@ def group( path=None, *, zarr_version=None, - meta_array=None + meta_array=None, ): """Create a group. @@ -1452,7 +1448,7 @@ def open_group( storage_options=None, *, zarr_version=None, - meta_array=None + meta_array=None, ): """Open a group using file-mode-like semantics. diff --git a/zarr/indexing.py b/zarr/indexing.py index 487cc8b9d9..3042147ebb 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -111,7 +111,6 @@ def is_pure_orthogonal_indexing(selection, ndim): def normalize_integer_selection(dim_sel, dim_len): - # normalize type to int dim_sel = int(dim_sel) @@ -145,7 +144,6 @@ def normalize_integer_selection(dim_sel, dim_len): class IntDimIndexer: def __init__(self, dim_sel, dim_len, dim_chunk_len): - # normalize dim_sel = normalize_integer_selection(dim_sel, dim_len) @@ -169,7 +167,6 @@ def ceildiv(a, b): class SliceDimIndexer: def __init__(self, dim_sel, dim_len, dim_chunk_len): - # normalize self.start, self.stop, self.step = dim_sel.indices(dim_len) if self.step < 1: @@ -182,14 +179,12 @@ def __init__(self, dim_sel, dim_len, dim_chunk_len): self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) def __iter__(self): - # figure out the range of chunks we need to visit dim_chunk_ix_from = self.start // self.dim_chunk_len dim_chunk_ix_to = ceildiv(self.stop, self.dim_chunk_len) # iterate over chunks in range for dim_chunk_ix in range(dim_chunk_ix_from, dim_chunk_ix_to): - # compute offsets for chunk within overall array dim_offset = dim_chunk_ix * self.dim_chunk_len dim_limit = min(self.dim_len, (dim_chunk_ix + 1) * self.dim_chunk_len) @@ -237,7 +232,6 @@ def check_selection_length(selection, shape): def replace_ellipsis(selection, shape): - selection = ensure_tuple(selection) # count number of ellipsis present @@ -330,14 +324,12 @@ def is_basic_selection(selection): # noinspection PyProtectedMember class BasicIndexer: def __init__(self, selection, array): - # handle ellipsis selection = replace_ellipsis(selection, array._shape) # setup per-dimension indexers dim_indexers = [] for dim_sel, dim_len, dim_chunk_len in zip(selection, array._shape, array._chunks): - if is_integer(dim_sel): dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) @@ -358,7 +350,6 @@ def __init__(self, selection, array): def __iter__(self): for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) out_selection = tuple( @@ -370,7 +361,6 @@ def __iter__(self): class BoolArrayDimIndexer: def __init__(self, dim_sel, dim_len, dim_chunk_len): - # check number of dimensions if not is_bool_array(dim_sel, 1): raise IndexError( @@ -402,10 +392,8 @@ def __init__(self, dim_sel, dim_len, dim_chunk_len): self.dim_chunk_ixs = np.nonzero(self.chunk_nitems)[0] def __iter__(self): - # iterate over chunks with at least one item for dim_chunk_ix in self.dim_chunk_ixs: - # find region in chunk dim_offset = dim_chunk_ix * self.dim_chunk_len dim_chunk_sel = self.dim_sel[dim_offset : dim_offset + self.dim_chunk_len] @@ -472,7 +460,6 @@ def __init__( boundscheck=True, order=Order.UNKNOWN, ): - # ensure 1d array dim_sel = np.asanyarray(dim_sel) if not is_integer_array(dim_sel, 1): @@ -526,9 +513,7 @@ def __init__( self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) def __iter__(self): - for dim_chunk_ix in self.dim_chunk_ixs: - # find region in output if dim_chunk_ix == 0: start = 0 @@ -602,7 +587,6 @@ def oindex_set(a, selection, value): # noinspection PyProtectedMember class OrthogonalIndexer: def __init__(self, selection, array): - # handle ellipsis selection = replace_ellipsis(selection, array._shape) @@ -612,7 +596,6 @@ def __init__(self, selection, array): # setup per-dimension indexers dim_indexers = [] for dim_sel, dim_len, dim_chunk_len in zip(selection, array._shape, array._chunks): - if is_integer(dim_sel): dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) @@ -649,7 +632,6 @@ def __init__(self, selection, array): def __iter__(self): for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) out_selection = tuple( @@ -658,7 +640,6 @@ def __iter__(self): # handle advanced indexing arrays orthogonally if self.is_advanced: - # N.B., numpy doesn't support orthogonal indexing directly as yet, # so need to work around via np.ix_. Also np.ix_ does not support a # mixture of arrays and slices or integers, so need to convert slices @@ -692,7 +673,6 @@ def __setitem__(self, selection, value): # noinspection PyProtectedMember class BlockIndexer: def __init__(self, selection, array): - # handle ellipsis selection = replace_ellipsis(selection, array._shape) @@ -794,7 +774,6 @@ def is_mask_selection(selection, array): # noinspection PyProtectedMember class CoordinateIndexer: def __init__(self, selection, array): - # some initial normalization selection = ensure_tuple(selection) selection = tuple([i] if is_integer(i) else i for i in selection) @@ -810,7 +789,6 @@ def __init__(self, selection, array): # handle wraparound, boundscheck for dim_sel, dim_len in zip(selection, array.shape): - # handle wraparound wraparound_indices(dim_sel, dim_len) @@ -861,10 +839,8 @@ def __init__(self, selection, array): self.chunk_mixs = np.unravel_index(self.chunk_rixs, array._cdata_shape) def __iter__(self): - # iterate over chunks for i, chunk_rix in enumerate(self.chunk_rixs): - chunk_coords = tuple(m[i] for m in self.chunk_mixs) if chunk_rix == 0: start = 0 @@ -891,7 +867,6 @@ def __iter__(self): # noinspection PyProtectedMember class MaskIndexer(CoordinateIndexer): def __init__(self, selection, array): - # some initial normalization selection = ensure_tuple(selection) selection = replace_lists(selection) diff --git a/zarr/meta.py b/zarr/meta.py index 48791ddf17..f23889f3ea 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -89,7 +89,6 @@ class Metadata2: @classmethod def parse_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]: - # Here we allow that a store may return an already-parsed metadata object, # or a string of JSON that we will parse here. We allow for an already-parsed # object to accommodate a consolidated metadata store, where all the metadata for diff --git a/zarr/n5.py b/zarr/n5.py index 7e73905527..44b44e69e2 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -72,21 +72,18 @@ class N5Store(NestedDirectoryStore): def __getitem__(self, key: str) -> bytes: if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) value = group_metadata_to_zarr(self._load_n5_attrs(key_new)) return json_dumps(value) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) top_level = key == zarr_array_meta_key value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level) return json_dumps(value) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) value = attrs_to_zarr(self._load_n5_attrs(key_new)) @@ -104,9 +101,7 @@ def __getitem__(self, key: str) -> bytes: return super().__getitem__(key_new) def __setitem__(self, key: str, value: Any): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) n5_attrs = self._load_n5_attrs(key_new) @@ -115,7 +110,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) top_level = key == zarr_array_meta_key n5_attrs = self._load_n5_attrs(key_new) @@ -123,7 +117,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) n5_attrs = self._load_n5_attrs(key_new) @@ -166,9 +159,7 @@ def __delitem__(self, key: str): super().__delitem__(key_new) def __contains__(self, key): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) if key_new not in self: return False @@ -176,18 +167,15 @@ def __contains__(self, key): return "dimensions" not in self._load_n5_attrs(key_new) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) # array if attributes contain 'dimensions' return "dimensions" in self._load_n5_attrs(key_new) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) return self._contains_attrs(key_new) elif is_chunk_key(key): - key_new = invert_chunk_coords(key) else: key_new = key @@ -198,7 +186,6 @@ def __eq__(self, other): return isinstance(other, N5Store) and self.path == other.path def listdir(self, path: Optional[str] = None): - if path is not None: path = invert_chunk_coords(path) path = cast(str, path) @@ -208,7 +195,6 @@ def listdir(self, path: Optional[str] = None): children = super().listdir(path=path) if self._is_array(path): - # replace n5 attribute file with respective zarr attribute files children.remove(n5_attrs_key) children.append(zarr_array_meta_key) @@ -234,7 +220,6 @@ def listdir(self, path: Optional[str] = None): return sorted(new_children) elif self._is_group(path): - # replace n5 attribute file with respective zarr attribute files children.remove(n5_attrs_key) children.append(zarr_group_meta_key) @@ -244,7 +229,6 @@ def listdir(self, path: Optional[str] = None): return sorted(children) else: - return children def _load_n5_attrs(self, path: str) -> Dict[str, Any]: @@ -255,7 +239,6 @@ def _load_n5_attrs(self, path: str) -> Dict[str, Any]: return {} def _is_group(self, path: str): - if path is None: attrs_key = n5_attrs_key else: @@ -265,7 +248,6 @@ def _is_group(self, path: str): return len(n5_attrs) > 0 and "dimensions" not in n5_attrs def _is_array(self, path: str): - if path is None: attrs_key = n5_attrs_key else: @@ -274,7 +256,6 @@ def _is_array(self, path: str): return "dimensions" in self._load_n5_attrs(attrs_key) def _contains_attrs(self, path: str): - if path is None: attrs_key = n5_attrs_key else: @@ -376,21 +357,18 @@ def _normalize_key(self, key: str): def __getitem__(self, key: str) -> bytes: if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) value = group_metadata_to_zarr(self._load_n5_attrs(key_new)) return json_dumps(value) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) top_level = key == zarr_array_meta_key value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level) return json_dumps(value) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) value = attrs_to_zarr(self._load_n5_attrs(key_new)) @@ -409,7 +387,6 @@ def __getitem__(self, key: str) -> bytes: def __setitem__(self, key: str, value: Any): if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) n5_attrs = self._load_n5_attrs(key_new) @@ -418,7 +395,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) top_level = key == zarr_array_meta_key n5_attrs = self._load_n5_attrs(key_new) @@ -427,7 +403,6 @@ def __setitem__(self, key: str, value: Any): value = json_dumps(n5_attrs) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) n5_attrs = self._load_n5_attrs(key_new) @@ -456,7 +431,6 @@ def __setitem__(self, key: str, value: Any): super().__setitem__(key_new, value) def __delitem__(self, key: str): - if key.endswith(zarr_group_meta_key): key_new = key.replace(zarr_group_meta_key, self._group_meta_key) elif key.endswith(zarr_array_meta_key): @@ -471,7 +445,6 @@ def __delitem__(self, key: str): def __contains__(self, key: Any): if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) if key_new not in self: return False @@ -479,13 +452,11 @@ def __contains__(self, key: Any): return "dimensions" not in self._load_n5_attrs(key_new) elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) # array if attributes contain 'dimensions' return "dimensions" in self._load_n5_attrs(key_new) elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) return self._contains_attrs(key_new) @@ -508,7 +479,6 @@ def listdir(self, path: Optional[str] = None): # doesn't provide. children = super().listdir(path=path) if self._is_array(path): - # replace n5 attribute file with respective zarr attribute files children.remove(self._array_meta_key) children.append(zarr_array_meta_key) @@ -532,7 +502,6 @@ def listdir(self, path: Optional[str] = None): return sorted(new_children) elif self._is_group(path): - # replace n5 attribute file with respective zarr attribute files children.remove(self._group_meta_key) children.append(zarr_group_meta_key) @@ -550,7 +519,6 @@ def _load_n5_attrs(self, path: str): return {} def _is_group(self, path: Optional[str]): - if path is None: attrs_key = self._attrs_key else: @@ -560,7 +528,6 @@ def _is_group(self, path: Optional[str]): return len(n5_attrs) > 0 and "dimensions" not in n5_attrs def _is_array(self, path: Optional[str]): - if path is None: attrs_key = self._attrs_key else: @@ -569,7 +536,6 @@ def _is_array(self, path: Optional[str]): return "dimensions" in self._load_n5_attrs(attrs_key) def _contains_attrs(self, path: Optional[str]): - if path is None: attrs_key = self._attrs_key else: @@ -712,7 +678,6 @@ def attrs_to_zarr(attrs: Dict[str, Any]) -> Dict[str, Any]: def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict[str, Any]: - if compressor_config is None: return {"type": "raw"} else: @@ -726,19 +691,16 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict n5_config = {"type": codec_id} if codec_id == "bz2": - n5_config["type"] = "bzip2" n5_config["blockSize"] = _compressor_config["level"] elif codec_id == "blosc": - n5_config["cname"] = _compressor_config["cname"] n5_config["clevel"] = _compressor_config["clevel"] n5_config["shuffle"] = _compressor_config["shuffle"] n5_config["blocksize"] = _compressor_config["blocksize"] elif codec_id == "lzma": - # Switch to XZ for N5 if we are using the default XZ format. # Note: 4 is the default, which is lzma.CHECK_CRC64. if _compressor_config["format"] == 1 and _compressor_config["check"] in [-1, 4]: @@ -760,50 +722,42 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict n5_config["preset"] = 6 elif codec_id == "zlib": - n5_config["type"] = "gzip" n5_config["level"] = _compressor_config["level"] n5_config["useZlib"] = True elif codec_id == "gzip": - n5_config["type"] = "gzip" n5_config["level"] = _compressor_config["level"] n5_config["useZlib"] = False else: - n5_config.update({k: v for k, v in _compressor_config.items() if k != "type"}) return n5_config def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dict[str, Any]]: - codec_id = compressor_config["type"] zarr_config = {"id": codec_id} if codec_id == "bzip2": - zarr_config["id"] = "bz2" zarr_config["level"] = compressor_config["blockSize"] elif codec_id == "blosc": - zarr_config["cname"] = compressor_config["cname"] zarr_config["clevel"] = compressor_config["clevel"] zarr_config["shuffle"] = compressor_config["shuffle"] zarr_config["blocksize"] = compressor_config["blocksize"] elif codec_id == "lzma": - zarr_config["format"] = compressor_config["format"] zarr_config["check"] = compressor_config["check"] zarr_config["preset"] = compressor_config["preset"] zarr_config["filters"] = compressor_config["filters"] elif codec_id == "xz": - zarr_config["id"] = "lzma" zarr_config["format"] = 1 # lzma.FORMAT_XZ zarr_config["check"] = -1 @@ -811,7 +765,6 @@ def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dic zarr_config["filters"] = None elif codec_id == "gzip": - if "useZlib" in compressor_config and compressor_config["useZlib"]: zarr_config["id"] = "zlib" zarr_config["level"] = compressor_config["level"] @@ -820,22 +773,18 @@ def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dic zarr_config["level"] = compressor_config["level"] elif codec_id == "raw": - return None else: - zarr_config.update({k: v for k, v in compressor_config.items() if k != "type"}) return zarr_config class N5ChunkWrapper(Codec): - codec_id = "n5_wrapper" def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None): - self.dtype = np.dtype(dtype) self.chunk_shape = tuple(chunk_shape) # is the dtype a little endian format? @@ -860,7 +809,6 @@ def get_config(self): return config def encode(self, chunk): - assert chunk.flags.c_contiguous header = self._create_header(chunk) @@ -872,12 +820,10 @@ def encode(self, chunk): return header + chunk.tobytes(order="A") def decode(self, chunk, out=None) -> bytes: - len_header, chunk_shape = self._read_header(chunk) chunk = chunk[len_header:] if out is not None: - # out should only be used if we read a complete chunk assert chunk_shape == self.chunk_shape, "Expected chunk of shape {}, found {}".format( self.chunk_shape, chunk_shape @@ -895,7 +841,6 @@ def decode(self, chunk, out=None) -> bytes: return out else: - if self._compressor: chunk = self._compressor.decode(chunk) @@ -915,7 +860,6 @@ def decode(self, chunk, out=None) -> bytes: @staticmethod def _create_header(chunk): - mode = struct.pack(">H", 0) num_dims = struct.pack(">H", len(chunk.shape)) shape = b"".join(struct.pack(">I", d) for d in chunk.shape[::-1]) @@ -924,7 +868,6 @@ def _create_header(chunk): @staticmethod def _read_header(chunk): - num_dims = struct.unpack(">H", chunk[2:4])[0] shape = tuple( struct.unpack(">I", chunk[i : i + 4])[0] for i in range(4, num_dims * 4 + 4, 4) diff --git a/zarr/storage.py b/zarr/storage.py index a7426e5345..585417f59c 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -483,7 +483,6 @@ def _init_array_metadata( dimension_separator=None, storage_transformers=(), ): - store_version = getattr(store, "_store_version", 2) path = normalize_storage_path(path) @@ -688,7 +687,6 @@ def _init_group_metadata( path: Optional[str] = None, chunk_store: Optional[StoreLike] = None, ): - store_version = getattr(store, "_store_version", 2) path = normalize_storage_path(path) @@ -1056,7 +1054,6 @@ class DirectoryStore(Store): """ def __init__(self, path, normalize_keys=False, dimension_separator=None): - # guard conditions path = os.path.abspath(path) if os.path.exists(path) and not os.path.isdir(path): @@ -1416,7 +1413,6 @@ def _normalize_key(self, key): def getitems( self, keys: Sequence[str], *, contexts: Mapping[str, Context] ) -> Mapping[str, Any]: - keys_transformed = [self._normalize_key(key) for key in keys] results = self.map.getitems(keys_transformed, on_error="omit") # The function calling this method may not recognize the transformed keys @@ -1770,7 +1766,6 @@ def __init__( mode="a", dimension_separator=None, ): - # store properties path = os.path.abspath(path) self.path = path diff --git a/zarr/tests/test_attrs.py b/zarr/tests/test_attrs.py index 7dd5b340a2..2d9553971b 100644 --- a/zarr/tests/test_attrs.py +++ b/zarr/tests/test_attrs.py @@ -30,7 +30,6 @@ def init_attributes(self, store, read_only=False, cache=True, zarr_version=2): return Attributes(store, key=root + "attrs", read_only=read_only, cache=cache) def test_storage(self, zarr_version): - store = _init_store(zarr_version) root = ".z" if zarr_version == 2 else meta_root attrs_key = root + "attrs" @@ -50,7 +49,6 @@ def test_storage(self, zarr_version): assert dict(foo="bar", baz=42) == d def test_utf8_encoding(self, zarr_version): - project_root = pathlib.Path(zarr.__file__).resolve().parent.parent fixdir = project_root / "fixture" testdir = fixdir / "utf8attrs" @@ -67,7 +65,6 @@ def test_utf8_encoding(self, zarr_version): assert fixture["utf8attrs"].attrs.asdict() == dict(foo="た") def test_get_set_del_contains(self, zarr_version): - store = _init_store(zarr_version) a = self.init_attributes(store, zarr_version=zarr_version) assert "foo" not in a @@ -84,7 +81,6 @@ def test_get_set_del_contains(self, zarr_version): a["foo"] def test_update_put(self, zarr_version): - store = _init_store(zarr_version) a = self.init_attributes(store, zarr_version=zarr_version) assert "foo" not in a @@ -102,7 +98,6 @@ def test_update_put(self, zarr_version): assert "baz" not in a def test_iterators(self, zarr_version): - store = _init_store(zarr_version) a = self.init_attributes(store, zarr_version=zarr_version) assert 0 == len(a) @@ -232,7 +227,6 @@ def test_caching_on(self, zarr_version): assert get_cnt == store.counter["__getitem__", attrs_key] def test_caching_off(self, zarr_version): - # setup store store = CountingDict() if zarr_version == 2 else CountingDictV3() attrs_key = ".zattrs" if zarr_version == 2 else "meta/root/attrs" diff --git a/zarr/tests/test_convenience.py b/zarr/tests/test_convenience.py index 389ce90a9d..7d190adc2c 100644 --- a/zarr/tests/test_convenience.py +++ b/zarr/tests/test_convenience.py @@ -57,7 +57,6 @@ def _init_creation_kwargs(zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) def test_open_array(path_type, zarr_version): - store = tempfile.mkdtemp() atexit.register(atexit_rmtree, store) store = path_type(store) @@ -86,7 +85,6 @@ def test_open_array(path_type, zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) def test_open_group(path_type, zarr_version): - store = tempfile.mkdtemp() atexit.register(atexit_rmtree, store) store = path_type(store) @@ -210,7 +208,6 @@ def test_tree(zarr_version): def test_consolidate_metadata( with_chunk_store, zarr_version, listable, monkeypatch, stores_from_path ): - # setup initial data if stores_from_path: store = tempfile.mkdtemp() @@ -399,7 +396,6 @@ def test_save_array_separator(tmpdir, options): class TestCopyStore(unittest.TestCase): - _version = 2 def setUp(self): @@ -536,7 +532,6 @@ def test_if_exists(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") class TestCopyStoreV3(TestCopyStore): - _version = 3 def setUp(self): @@ -557,7 +552,6 @@ def test_mismatched_store_versions(self): def check_copied_array(original, copied, without_attrs=False, expect_props=None): - # setup source_h5py = original.__module__.startswith("h5py.") dest_h5py = copied.__module__.startswith("h5py.") @@ -621,7 +615,6 @@ def check_copied_array(original, copied, without_attrs=False, expect_props=None) def check_copied_group(original, copied, without_attrs=False, expect_props=None, shallow=False): - # setup if expect_props is None: expect_props = dict() diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index b44c6379fd..8e586abfff 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -74,7 +74,6 @@ def _init_creation_kwargs(zarr_version, at_root=True): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_array(zarr_version, at_root): - expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version kwargs = _init_creation_kwargs(zarr_version, at_root) @@ -213,7 +212,6 @@ def test_full_additional_dtypes(zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_open_array(zarr_version, at_root, dimension_separator): - store = "data/array.zarr" kwargs = _init_creation_kwargs(zarr_version, at_root) @@ -329,7 +327,6 @@ def test_open_array(zarr_version, at_root, dimension_separator): def test_open_array_none(): - # open with both store and zarr_version = None z = open_array(mode="w", shape=100, chunks=10) assert isinstance(z, Array) @@ -339,7 +336,6 @@ def test_open_array_none(): @pytest.mark.parametrize("dimension_separator", [".", "/", None]) @pytest.mark.parametrize("zarr_version", _VERSIONS2) def test_open_array_infer_separator_from_store(zarr_version, dimension_separator): - if zarr_version == 3: StoreClass = DirectoryStoreV3 path = "data" @@ -370,7 +366,6 @@ def test_open_array_infer_separator_from_store(zarr_version, dimension_separator # TODO: N5 support for v3 @pytest.mark.parametrize("zarr_version", [None, 2]) def test_open_array_n5(zarr_version): - store = "data/array.zarr" kwargs = _init_creation_kwargs(zarr_version) @@ -409,7 +404,6 @@ def test_open_array_n5(zarr_version): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_open_array_dict_store(zarr_version, at_root): - # dict will become a KVStore store = dict() kwargs = _init_creation_kwargs(zarr_version, at_root) @@ -503,7 +497,6 @@ def test_empty_like(zarr_version, at_root): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_zeros_like(zarr_version, at_root): - kwargs = _init_creation_kwargs(zarr_version, at_root) expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version @@ -529,7 +522,6 @@ def test_zeros_like(zarr_version, at_root): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_ones_like(zarr_version, at_root): - kwargs = _init_creation_kwargs(zarr_version, at_root) expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version @@ -556,7 +548,6 @@ def test_ones_like(zarr_version, at_root): @pytest.mark.parametrize("zarr_version", _VERSIONS) @pytest.mark.parametrize("at_root", [False, True]) def test_full_like(zarr_version, at_root): - kwargs = _init_creation_kwargs(zarr_version, at_root) expected_zarr_version = DEFAULT_ZARR_VERSION if zarr_version is None else zarr_version diff --git a/zarr/tests/test_dim_separator.py b/zarr/tests/test_dim_separator.py index 987852dfd0..0a5814e65f 100644 --- a/zarr/tests/test_dim_separator.py +++ b/zarr/tests/test_dim_separator.py @@ -46,7 +46,6 @@ def dataset(tmpdir, request): static = project_root / "fixture" / suffix if not static.exists(): # pragma: no cover - if "nested" in which: # No way to reproduce the nested_legacy file via code generator = NestedDirectoryStore diff --git a/zarr/tests/test_filters.py b/zarr/tests/test_filters.py index d55be9145f..fc63cdca8d 100644 --- a/zarr/tests/test_filters.py +++ b/zarr/tests/test_filters.py @@ -30,7 +30,6 @@ def test_array_with_delta_filter(): - # setup astype = "u1" dtype = "i8" @@ -38,7 +37,6 @@ def test_array_with_delta_filter(): data = np.arange(100, dtype=dtype) for compressor in compressors: - a = array(data, chunks=10, compressor=compressor, filters=filters) # check round-trip @@ -57,7 +55,6 @@ def test_array_with_delta_filter(): def test_array_with_astype_filter(): - # setup encode_dtype = "i1" decode_dtype = "i8" @@ -68,7 +65,6 @@ def test_array_with_astype_filter(): data = np.arange(shape, dtype=decode_dtype) for compressor in compressors: - a = array(data, chunks=chunks, compressor=compressor, filters=filters) # check round-trip @@ -88,7 +84,6 @@ def test_array_with_astype_filter(): def test_array_with_scaleoffset_filter(): - # setup astype = "u1" dtype = "f8" @@ -97,7 +92,6 @@ def test_array_with_scaleoffset_filter(): data = np.linspace(1000, 1001, 34, dtype="f8") for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip @@ -116,7 +110,6 @@ def test_array_with_scaleoffset_filter(): def test_array_with_quantize_filter(): - # setup dtype = "f8" digits = 3 @@ -125,7 +118,6 @@ def test_array_with_quantize_filter(): data = np.linspace(0, 1, 34, dtype=dtype) for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip @@ -144,14 +136,12 @@ def test_array_with_quantize_filter(): def test_array_with_packbits_filter(): - # setup flt = PackBits() filters = [flt] data = np.random.randint(0, 2, size=100, dtype=bool) for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip @@ -170,14 +160,12 @@ def test_array_with_packbits_filter(): def test_array_with_categorize_filter(): - # setup data = np.random.choice(["foo", "bar", "baz"], size=100) flt = Categorize(dtype=data.dtype, labels=["foo", "bar", "baz"]) filters = [flt] for compressor in compressors: - a = array(data, chunks=5, compressor=compressor, filters=filters) # check round-trip diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index cbf59c55c3..6c08d7b88a 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1085,7 +1085,6 @@ def test_paths(self): g1.store.close() def test_pickle(self): - # setup group g = self.create_group() d = g.create_dataset("foo/bar", shape=100, chunks=10) @@ -1113,7 +1112,6 @@ def test_pickle(self): g2.store.close() def test_context_manager(self): - with self.create_group() as g: d = g.create_dataset("foo/bar", shape=100, chunks=10) d[:] = np.arange(100) @@ -1375,7 +1373,6 @@ def create_store(): return store, None def test_context_manager(self): - with self.create_group() as g: store = g.store d = g.create_dataset("foo/bar", shape=100, chunks=10) diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index 8a34c1e715..f10360e8b7 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -17,7 +17,6 @@ def test_normalize_integer_selection(): - assert 1 == normalize_integer_selection(1, 100) assert 99 == normalize_integer_selection(-1, 100) with pytest.raises(IndexError): @@ -29,7 +28,6 @@ def test_normalize_integer_selection(): def test_replace_ellipsis(): - # 1D, single item assert (0,) == replace_ellipsis(0, (100,)) @@ -68,7 +66,6 @@ def test_replace_ellipsis(): def test_get_basic_selection_0d(): - # setup a = np.array(42) z = zarr.create(shape=a.shape, dtype=a.dtype, fill_value=None) @@ -191,7 +188,6 @@ def _test_get_basic_selection(a, z, selection): # noinspection PyStatementEffect def test_get_basic_selection_1d(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -264,7 +260,6 @@ def test_get_basic_selection_1d(): # noinspection PyStatementEffect def test_get_basic_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -423,7 +418,6 @@ def test_fancy_indexing_doesnt_mix_with_implicit_slicing(): def test_set_basic_selection_0d(): - # setup v = np.array(42) a = np.zeros_like(v) @@ -479,7 +473,6 @@ def _test_get_orthogonal_selection(a, z, selection): # noinspection PyStatementEffect def test_get_orthogonal_selection_1d_bool(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -502,7 +495,6 @@ def test_get_orthogonal_selection_1d_bool(): # noinspection PyStatementEffect def test_get_orthogonal_selection_1d_int(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -561,7 +553,6 @@ def _test_get_orthogonal_selection_2d(a, z, ix0, ix1): # noinspection PyStatementEffect def test_get_orthogonal_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -570,7 +561,6 @@ def test_get_orthogonal_selection_2d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -641,7 +631,6 @@ def _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2): def test_get_orthogonal_selection_3d(): - # setup a = np.arange(100000, dtype=int).reshape(200, 50, 10) z = zarr.create(shape=a.shape, chunks=(60, 20, 3), dtype=a.dtype) @@ -650,7 +639,6 @@ def test_get_orthogonal_selection_3d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -673,7 +661,6 @@ def test_get_orthogonal_selection_3d(): def test_orthogonal_indexing_edge_cases(): - a = np.arange(6).reshape(1, 2, 3) z = zarr.create(shape=a.shape, chunks=(1, 2, 3), dtype=a.dtype) z[:] = a @@ -706,7 +693,6 @@ def _test_set_orthogonal_selection(v, a, z, selection): def test_set_orthogonal_selection_1d(): - # setup v = np.arange(1050, dtype=int) a = np.empty(v.shape, dtype=int) @@ -715,7 +701,6 @@ def test_set_orthogonal_selection_1d(): # test with different degrees of sparseness np.random.seed(42) for p in 0.5, 0.1, 0.01: - # boolean arrays ix = np.random.binomial(1, p, size=a.shape[0]).astype(bool) _test_set_orthogonal_selection(v, a, z, ix) @@ -734,7 +719,6 @@ def test_set_orthogonal_selection_1d(): def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1): - selections = [ # index both axes with array (ix0, ix1), @@ -749,7 +733,6 @@ def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1): def test_set_orthogonal_selection_2d(): - # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -758,7 +741,6 @@ def test_set_orthogonal_selection_2d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -780,7 +762,6 @@ def test_set_orthogonal_selection_2d(): def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2): - selections = ( # single value (84, 42, 4), @@ -807,7 +788,6 @@ def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2): def test_set_orthogonal_selection_3d(): - # setup v = np.arange(100000, dtype=int).reshape(200, 50, 10) a = np.empty_like(v) @@ -816,7 +796,6 @@ def test_set_orthogonal_selection_3d(): np.random.seed(42) # test with different degrees of sparseness for p in 0.5, 0.1, 0.01: - # boolean arrays ix0 = np.random.binomial(1, p, size=a.shape[0]).astype(bool) ix1 = np.random.binomial(1, 0.5, size=a.shape[1]).astype(bool) @@ -888,7 +867,6 @@ def _test_get_coordinate_selection(a, z, selection): # noinspection PyStatementEffect def test_get_coordinate_selection_1d(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -932,7 +910,6 @@ def test_get_coordinate_selection_1d(): def test_get_coordinate_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -1027,7 +1004,6 @@ def test_set_coordinate_selection_1d(): def test_set_coordinate_selection_2d(): - # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -1258,7 +1234,6 @@ def _test_get_mask_selection(a, z, selection): # noinspection PyStatementEffect def test_get_mask_selection_1d(): - # setup a = np.arange(1050, dtype=int) z = zarr.create(shape=a.shape, chunks=100, dtype=a.dtype) @@ -1285,7 +1260,6 @@ def test_get_mask_selection_1d(): # noinspection PyStatementEffect def test_get_mask_selection_2d(): - # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr.create(shape=a.shape, chunks=(300, 3), dtype=a.dtype) @@ -1318,7 +1292,6 @@ def _test_set_mask_selection(v, a, z, selection): def test_set_mask_selection_1d(): - # setup v = np.arange(1050, dtype=int) a = np.empty_like(v) @@ -1338,7 +1311,6 @@ def test_set_mask_selection_1d(): def test_set_mask_selection_2d(): - # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -1352,7 +1324,6 @@ def test_set_mask_selection_2d(): def test_get_selection_out(): - # basic selections a = np.arange(1050) z = zarr.create(shape=1050, chunks=100, dtype=a.dtype) @@ -1426,7 +1397,6 @@ def test_get_selection_out(): def test_get_selections_with_fields(): - a = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] a = np.array(a, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) z = zarr.create(shape=a.shape, chunks=2, dtype=a.dtype, fill_value=None) @@ -1444,7 +1414,6 @@ def test_get_selections_with_fields(): ] for fields in fields_fixture: - # total selection expect = a[fields] actual = z.get_basic_selection(Ellipsis, fields=fields) @@ -1534,7 +1503,6 @@ def test_get_selections_with_fields(): def test_set_selections_with_fields(): - v = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] v = np.array(v, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) a = np.empty_like(v) @@ -1553,7 +1521,6 @@ def test_set_selections_with_fields(): ] for fields in fields_fixture: - # currently multi-field assignment is not supported in numpy, so we won't support # it either if isinstance(fields, list) and len(fields) > 1: @@ -1567,7 +1534,6 @@ def test_set_selections_with_fields(): z.set_mask_selection([True, False, True], v, fields=fields) else: - if isinstance(fields, list) and len(fields) == 1: # work around numpy does not support multi-field assignment even if there # is only one field @@ -1752,7 +1718,6 @@ def test_accessed_chunks(shape, chunks, ops): z = zarr.create(shape=shape, chunks=chunks, store=store) for ii, (optype, slices) in enumerate(ops): - # Resolve the slices into the accessed chunks for each dimension chunks_per_dim = [] for N, C, sl in zip(shape, chunks, slices): diff --git a/zarr/tests/test_info.py b/zarr/tests/test_info.py index 7fb6feb11b..96eae999f4 100644 --- a/zarr/tests/test_info.py +++ b/zarr/tests/test_info.py @@ -7,7 +7,6 @@ @pytest.mark.parametrize("array_size", [10, 15000]) def test_info(array_size): - # setup g = zarr.group(store=dict(), chunk_store=dict(), synchronizer=zarr.ThreadSynchronizer()) g.create_group("foo") diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index db50560c8e..3e1e0f9d63 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -34,7 +34,6 @@ def assert_json_equal(expect, actual): def test_encode_decode_array_1(): - meta = dict( shape=(100,), chunks=(10,), @@ -76,7 +75,6 @@ def test_encode_decode_array_1(): def test_encode_decode_array_2(): - # some variations df = Delta(astype=" Tupl def normalize_dtype(dtype: Union[str, np.dtype], object_codec) -> Tuple[np.dtype, Any]: - # convenience API for object arrays if inspect.isclass(dtype): dtype = dtype.__name__ # type: ignore @@ -245,7 +244,6 @@ def is_total_slice(item, shape: Tuple[int]) -> bool: def normalize_resize_args(old_shape, *args): - # normalize new shape argument if len(args) == 1: new_shape = args[0] @@ -294,7 +292,6 @@ def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]: def normalize_fill_value(fill_value, dtype: np.dtype): - if fill_value is None or dtype.hasobject: # no fill value pass @@ -332,7 +329,6 @@ def normalize_fill_value(fill_value, dtype: np.dtype): def normalize_storage_path(path: Union[str, bytes, None]) -> str: - # handle bytes if isinstance(path, bytes): path = str(path, "ascii") @@ -342,7 +338,6 @@ def normalize_storage_path(path: Union[str, bytes, None]) -> str: path = str(path) if path: - # convert backslash to forward slash path = path.replace("\\", "/") @@ -506,7 +501,6 @@ def tree_widget(group, expand, level): class TreeViewer: def __init__(self, group, expand=False, level=None): - self.group = group self.expand = expand self.level = level From 54e31e9814a41cd7fd81255695971ce5e700ee3e Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:29:28 +0100 Subject: [PATCH 0005/1043] Use list comprehension where applicable (#1555) Even if this is only a test, list comprehensions are faster than repeatedly call append(). Also use tuple instead of list when possible. Co-authored-by: Davis Bennett --- zarr/tests/test_indexing.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index f10360e8b7..af046e9d28 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -1719,17 +1719,15 @@ def test_accessed_chunks(shape, chunks, ops): for ii, (optype, slices) in enumerate(ops): # Resolve the slices into the accessed chunks for each dimension - chunks_per_dim = [] - for N, C, sl in zip(shape, chunks, slices): - chunk_ind = np.arange(N, dtype=int)[sl] // C - chunks_per_dim.append(np.unique(chunk_ind)) + chunks_per_dim = [ + np.unique(np.arange(N, dtype=int)[sl] // C) for N, C, sl in zip(shape, chunks, slices) + ] # Combine and generate the cartesian product to determine the chunks keys that # will be accessed - chunks_accessed = [] - for comb in itertools.product(*chunks_per_dim): - chunks_accessed.append(".".join([str(ci) for ci in comb])) - + chunks_accessed = ( + ".".join([str(ci) for ci in comb]) for comb in itertools.product(*chunks_per_dim) + ) counts_before = store.counter.copy() # Perform the operation From 7d2c9bf5ce4c998d95630d9a1202e27e58926838 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:35:23 +0000 Subject: [PATCH 0006/1043] Bump numcodecs from 0.11.0 to 0.12.1 (#1580) Bumps [numcodecs](https://github.com/zarr-developers/numcodecs) from 0.11.0 to 0.12.1. - [Release notes](https://github.com/zarr-developers/numcodecs/releases) - [Changelog](https://github.com/zarr-developers/numcodecs/blob/main/docs/release.rst) - [Commits](https://github.com/zarr-developers/numcodecs/compare/v0.11.0...v0.12.1) --- updated-dependencies: - dependency-name: numcodecs dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index e2be6eb825..afea816d87 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -1,7 +1,7 @@ # library requirements asciitree==0.3.3 fasteners==0.19 -numcodecs==0.11.0 +numcodecs==0.12.1 msgpack-python==0.5.6 setuptools-scm==8.0.4 # test requirements From 10dee6ba0c0ce6ab29333e7a50f0afa4f6de06ca Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:40:21 +0100 Subject: [PATCH 0007/1043] Use format specification mini-language to format string (#1558) Co-authored-by: Joe Hamman --- zarr/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/storage.py b/zarr/storage.py index 585417f59c..5ba8071395 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -784,7 +784,7 @@ def __len__(self): return len(self._mutable_mapping) def __repr__(self): - return f"<{self.__class__.__name__}: \n{repr(self._mutable_mapping)}\n at {hex(id(self))}>" + return f"<{self.__class__.__name__}: \n{self._mutable_mapping!r}\n at {id(self):#x}>" def __eq__(self, other): if isinstance(other, KVStore): From 40a6e817b17e1fe600b188478ba38fb6978a5273 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 7 Dec 2023 22:45:50 +0100 Subject: [PATCH 0008/1043] Single startswith() call instead of multiple ones (#1556) It's faster and probably more readable. Co-authored-by: Davis Bennett Co-authored-by: Joe Hamman --- zarr/_storage/store.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 80e4ad8f75..667ca38147 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -221,9 +221,8 @@ def _validate_key(self, key: str): ) if ( - not key.startswith("data/") - and (not key.startswith("meta/")) - and (not key == "zarr.json") + not key.startswith(("data/", "meta/")) + and key != "zarr.json" # TODO: Possibly allow key == ".zmetadata" too if we write a # consolidated metadata spec corresponding to this? ): From 5954ff95803c1343d022f6181ed397c7095f4a0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 21:46:14 +0000 Subject: [PATCH 0009/1043] Bump pymongo from 4.5.0 to 4.6.1 (#1585) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.5.0 to 4.6.1. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.5.0...4.6.1) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index f3ea80a546..5a3340a282 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.16.0 # pyup: ignore redis==5.0.1 types-redis types-setuptools -pymongo==4.5.0 +pymongo==4.6.1 # optional test requirements coverage pytest-cov==4.1.0 From 6ad7b0e2ddabdcc5087e23f003edf123d21e9a25 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 8 Dec 2023 00:07:10 +0100 Subject: [PATCH 0010/1043] Move codespell options around (#1196) Starting with codespell 2.2.2, options can be specified in `pyrpoject.toml` in addition to `setup.cfg`: https://github.com/codespell-project/codespell#using-a-config-file Specifying options in a config file instead of command line options in `.pre-commit-config.yaml` ensures codespell uses the same options when run as pre-commit hook or from the command line in the repository root directory. --- .pre-commit-config.yaml | 1 - pyproject.toml | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e985d24000..029dcda58f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,6 @@ repos: rev: v2.2.5 hooks: - id: codespell - args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 22ea19f28f..36a0d896ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,3 +137,8 @@ filterwarnings = [ "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", ] + + +[tool.codespell] +ignore-words-list = "ba,ihs,kake,nd,noe,nwo,te,fo,zar" +skip = 'fixture,.git' From cf32382b9a228eaaafe30ab82d05b9303824a783 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Dec 2023 15:19:14 +0100 Subject: [PATCH 0011/1043] Bump fsspec from 2023.10.0 to 2023.12.1 (#1600) * Bump fsspec from 2023.10.0 to 2023.12.1 Bumps [fsspec](https://github.com/fsspec/filesystem_spec) from 2023.10.0 to 2023.12.1. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2023.10.0...2023.12.1) --- updated-dependencies: - dependency-name: fsspec dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update s3fs as well * Fix s3fs --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- requirements_dev_optional.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 5a3340a282..13385a243a 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -18,6 +18,6 @@ pytest-cov==4.1.0 pytest-doctestplus==1.0.0 pytest-timeout==2.2.0 h5py==3.10.0 -fsspec==2023.10.0 -s3fs==2023.10.0 +fsspec==2023.12.1 +s3fs==2023.12.1 moto[server]>=4.0.8 From 4d79cfc84f7f3914a04d9468666685520cc21276 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 8 Dec 2023 16:41:51 +0000 Subject: [PATCH 0012/1043] Add type hints to zarr.create (#1536) * Add type hints to zarr.create * Use protocol for MetaArray * Use protocol for Synchronizer * Fix Path typing * Add release note * Fix dim separator typing * Ignore ... in coverage reporting * Fix chunk typing --------- Co-authored-by: Davis Bennett --- docs/release.rst | 6 ++++++ pyproject.toml | 1 + zarr/_storage/store.py | 3 ++- zarr/creation.py | 46 +++++++++++++++++++++++------------------ zarr/storage.py | 2 +- zarr/sync.py | 12 +++++++++-- zarr/tests/test_core.py | 8 ++++--- zarr/types.py | 13 ++++++++++++ zarr/util.py | 5 +++-- 9 files changed, 67 insertions(+), 29 deletions(-) create mode 100644 zarr/types.py diff --git a/docs/release.rst b/docs/release.rst index 842c36e290..c18e0b8c20 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,12 @@ Release notes Unreleased ---------- +Enhancements +~~~~~~~~~~~~ + +* Added type hints to ``zarr.creation.create()``. + By :user:`David Stansby ` :issue:`1536`. + Docs ~~~~ diff --git a/pyproject.toml b/pyproject.toml index 36a0d896ea..4b7fef6003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ Homepage = "https://github.com/zarr-developers/zarr-python" exclude_lines = [ "pragma: no cover", "pragma: ${PY_MAJOR_VERSION} no cover", + '.*\.\.\.' # Ignore "..." lines ] [tool.coverage.run] diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 667ca38147..09f0b68602 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -9,6 +9,7 @@ from zarr.meta import Metadata2, Metadata3 from zarr.util import normalize_storage_path from zarr.context import Context +from zarr.types import ZARR_VERSION # v2 store keys array_meta_key = ".zarray" @@ -19,7 +20,7 @@ meta_root = "meta/root/" data_root = "data/root/" -DEFAULT_ZARR_VERSION = 2 +DEFAULT_ZARR_VERSION: ZARR_VERSION = 2 v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"] diff --git a/zarr/creation.py b/zarr/creation.py index 6227f90b7b..d4f570895a 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -1,7 +1,10 @@ -from typing import Optional +from collections.abc import MutableMapping +from typing import Optional, Tuple, Union, Sequence from warnings import warn import numpy as np +import numpy.typing as npt +from numcodecs.abc import Codec from numcodecs.registry import codec_registry from zarr._storage.store import DEFAULT_ZARR_VERSION @@ -19,32 +22,35 @@ normalize_storage_path, normalize_store_arg, ) +from zarr._storage.store import StorageTransformer +from zarr.sync import Synchronizer +from zarr.types import ZARR_VERSION, DIMENSION_SEPARATOR, MEMORY_ORDER, MetaArray, PathLike from zarr.util import normalize_dimension_separator def create( - shape, - chunks=True, - dtype=None, + shape: Union[int, Tuple[int, ...]], + chunks: Union[int, Tuple[int, ...], bool] = True, + dtype: Optional[npt.DTypeLike] = None, compressor="default", fill_value: Optional[int] = 0, - order="C", - store=None, - synchronizer=None, - overwrite=False, - path=None, - chunk_store=None, - filters=None, - cache_metadata=True, - cache_attrs=True, - read_only=False, - object_codec=None, - dimension_separator=None, - write_empty_chunks=True, + order: MEMORY_ORDER = "C", + store: Optional[Union[str, MutableMapping]] = None, + synchronizer: Optional[Synchronizer] = None, + overwrite: bool = False, + path: Optional[PathLike] = None, + chunk_store: Optional[MutableMapping] = None, + filters: Optional[Sequence[Codec]] = None, + cache_metadata: bool = True, + cache_attrs: bool = True, + read_only: bool = False, + object_codec: Optional[Codec] = None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, + write_empty_chunks: bool = True, *, - zarr_version=None, - meta_array=None, - storage_transformers=(), + zarr_version: Optional[ZARR_VERSION] = None, + meta_array: Optional[MetaArray] = None, + storage_transformers: Sequence[StorageTransformer] = (), **kwargs, ): """Create an array. diff --git a/zarr/storage.py b/zarr/storage.py index 5ba8071395..1c3b39862a 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -40,6 +40,7 @@ from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like from numcodecs.registry import codec_registry from zarr.context import Context +from zarr.types import PathLike as Path from zarr.errors import ( MetadataError, @@ -105,7 +106,6 @@ default_compressor = Zlib() -Path = Union[str, bytes, None] # allow MutableMapping for backwards compatibility StoreLike = Union[BaseStore, MutableMapping] diff --git a/zarr/sync.py b/zarr/sync.py index 49684a51ee..2e843f6557 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -1,11 +1,19 @@ import os from collections import defaultdict from threading import Lock +from typing import Protocol import fasteners -class ThreadSynchronizer: +class Synchronizer(Protocol): + """Base class for synchronizers.""" + + def __getitem__(self, item): + ... + + +class ThreadSynchronizer(Synchronizer): """Provides synchronization using thread locks.""" def __init__(self): @@ -24,7 +32,7 @@ def __setstate__(self, *args): self.__init__() -class ProcessSynchronizer: +class ProcessSynchronizer(Synchronizer): """Provides synchronization using file locks via the `fasteners `_ package. diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index f3ca73dea8..a3fde4050d 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3,7 +3,7 @@ import sys import pickle import shutil -from typing import Any, Literal, Optional, Tuple, Union +from typing import Any, Literal, Optional, Tuple, Union, Sequence import unittest from itertools import zip_longest from tempfile import mkdtemp @@ -26,6 +26,7 @@ VLenUTF8, Zlib, ) +from numcodecs.abc import Codec from numcodecs.compat import ensure_bytes, ensure_ndarray from numcodecs.tests.common import greetings from numpy.testing import assert_array_almost_equal, assert_array_equal @@ -73,6 +74,7 @@ from zarr.tests.test_storage_v3 import DummyStorageTransfomer from zarr.util import buffer_size from zarr.tests.util import abs_container, skip_test_env_var, have_fsspec, mktemp +from zarr.types import DIMENSION_SEPARATOR # noinspection PyMethodMayBeStatic @@ -82,8 +84,8 @@ class TestArray: root = "" path = "" compressor = Zlib(level=1) - filters = None - dimension_separator: Literal["/", ".", None] = None + filters: Optional[Sequence[Codec]] = None + dimension_separator: Optional[DIMENSION_SEPARATOR] = None cache_metadata = True cache_attrs = True partial_decompress: bool = False diff --git a/zarr/types.py b/zarr/types.py new file mode 100644 index 0000000000..1de270f25c --- /dev/null +++ b/zarr/types.py @@ -0,0 +1,13 @@ +from typing import Literal, Protocol, Union + +ZARR_VERSION = Literal[2, 3] +DIMENSION_SEPARATOR = Literal[".", "/"] +MEMORY_ORDER = Literal["C", "F"] + + +PathLike = Union[str, bytes, None] + + +class MetaArray(Protocol): + def __array_function__(self, func, types, args, kwargs): + ... diff --git a/zarr/util.py b/zarr/util.py index df1cd9d409..f97094b93a 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -31,6 +31,7 @@ from numcodecs.ndarray_like import NDArrayLike from numcodecs.registry import codec_registry from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo +from zarr.types import DIMENSION_SEPARATOR KeyType = TypeVar("KeyType") ValueType = TypeVar("ValueType") @@ -284,9 +285,9 @@ def normalize_order(order: str) -> str: return order -def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]: +def normalize_dimension_separator(sep: Optional[str]) -> Optional[DIMENSION_SEPARATOR]: if sep in (".", "/", None): - return sep + return cast(Optional[DIMENSION_SEPARATOR], sep) else: raise ValueError("dimension_separator must be either '.' or '/', found: %r" % sep) From 12abd4e434e816e9b8f19b1ceb89438fa4269737 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 18 Dec 2023 11:57:44 +0000 Subject: [PATCH 0013/1043] Remove unused mypy ignore comments (#1602) Co-authored-by: Davis Bennett --- pyproject.toml | 5 +++-- zarr/_storage/store.py | 4 ++-- zarr/_storage/v3_storage_transformers.py | 2 +- zarr/meta.py | 4 ++-- zarr/storage.py | 12 ++++++------ zarr/util.py | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4b7fef6003..33e8573830 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,9 +120,10 @@ exclude = ''' ''' [tool.mypy] -python_version = "3.8" ignore_missing_imports = true -follow_imports = "silent" +warn_unused_configs = true +warn_redundant_casts = true +warn_unused_ignores = true [tool.pytest.ini_options] doctest_optionflags = [ diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 09f0b68602..36b596769a 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -642,10 +642,10 @@ def _rmdir_from_keys_v3(store: StoreV3, path: str = "") -> None: sfx = _get_metadata_suffix(store) array_meta_file = meta_dir + ".array" + sfx if array_meta_file in store: - store.erase(array_meta_file) # type: ignore + store.erase(array_meta_file) group_meta_file = meta_dir + ".group" + sfx if group_meta_file in store: - store.erase(group_meta_file) # type: ignore + store.erase(group_meta_file) def _listdir_from_keys(store: BaseStore, path: Optional[str] = None) -> List[str]: diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index ff31a7281c..3afc3823a3 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -351,7 +351,7 @@ def erase_prefix(self, prefix): def rmdir(self, path=None): path = normalize_storage_path(path) - _rmdir_from_keys_v3(self, path) # type: ignore + _rmdir_from_keys_v3(self, path) def __contains__(self, key): if self._is_data_key(key): diff --git a/zarr/meta.py b/zarr/meta.py index f23889f3ea..d9797e4754 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -234,8 +234,8 @@ def decode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> return np.array(v, dtype=dtype)[()] elif dtype.kind in "c": v = ( - cls.decode_fill_value(v[0], dtype.type().real.dtype), # type: ignore - cls.decode_fill_value(v[1], dtype.type().imag.dtype), # type: ignore + cls.decode_fill_value(v[0], dtype.type().real.dtype), + cls.decode_fill_value(v[1], dtype.type().imag.dtype), ) v = v[0] + 1j * v[1] return np.array(v, dtype=dtype)[()] diff --git a/zarr/storage.py b/zarr/storage.py index 1c3b39862a..aa27e98e6f 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -206,7 +206,7 @@ def rmdir(store: StoreLike, path: Path = None): store_version = getattr(store, "_store_version", 2) if hasattr(store, "rmdir") and store.is_erasable(): # type: ignore # pass through - store.rmdir(path) # type: ignore + store.rmdir(path) else: # slow version, delete one key at a time if store_version == 2: @@ -236,7 +236,7 @@ def listdir(store: BaseStore, path: Path = None): path = normalize_storage_path(path) if hasattr(store, "listdir"): # pass through - return store.listdir(path) # type: ignore + return store.listdir(path) else: # slow version, iterate through all keys warnings.warn( @@ -289,7 +289,7 @@ def getsize(store: BaseStore, path: Path = None) -> int: if hasattr(store, "getsize"): # pass through path = normalize_storage_path(path) - return store.getsize(path) # type: ignore + return store.getsize(path) elif isinstance(store, MutableMapping): return _getsize(store, path) else: @@ -627,7 +627,7 @@ def _init_array_metadata( key = _prefix_to_array_key(store, _path_to_prefix(path)) if hasattr(store, "_metadata_class"): - store[key] = store._metadata_class.encode_array_metadata(meta) # type: ignore + store[key] = store._metadata_class.encode_array_metadata(meta) else: store[key] = encode_array_metadata(meta) @@ -730,10 +730,10 @@ def _init_group_metadata( if store_version == 3: meta = {"attributes": {}} # type: ignore else: - meta = {} # type: ignore + meta = {} key = _prefix_to_group_key(store, _path_to_prefix(path)) if hasattr(store, "_metadata_class"): - store[key] = store._metadata_class.encode_group_metadata(meta) # type: ignore + store[key] = store._metadata_class.encode_group_metadata(meta) else: store[key] = encode_group_metadata(meta) diff --git a/zarr/util.py b/zarr/util.py index f97094b93a..54c389db69 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -183,7 +183,7 @@ def normalize_chunks(chunks: Any, shape: Tuple[int, ...], typesize: int) -> Tupl def normalize_dtype(dtype: Union[str, np.dtype], object_codec) -> Tuple[np.dtype, Any]: # convenience API for object arrays if inspect.isclass(dtype): - dtype = dtype.__name__ # type: ignore + dtype = dtype.__name__ if isinstance(dtype, str): # allow ':' to delimit class from codec arguments tokens = dtype.split(":") From c2f5f0058d25eaa6695334e399b7b3a2d23f7a10 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 20:47:31 -0700 Subject: [PATCH 0014/1043] Bump actions/setup-python from 4.7.1 to 5.0.0 (#1605) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 3bd25bfbf7..8d8512294d 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v4.7.1 + - uses: actions/setup-python@v5.0.0 name: Install Python with: python-version: '3.8' From 490e0fe4e59f234cde85b103252acefa34927184 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 22:01:40 +0100 Subject: [PATCH 0015/1043] Bump github/codeql-action from 2 to 3 (#1609) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7013f1784f..bb3d433629 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -42,7 +42,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -56,7 +56,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -69,4 +69,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From b5f79ddfe7821cc9387fc4084bd7672f59215400 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:38:04 -0700 Subject: [PATCH 0016/1043] chore: update pre-commit hooks (#1448) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pre-commit hooks updates: - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.0.224 → v0.1.8](https://github.com/astral-sh/ruff-pre-commit/compare/v0.0.224...v0.1.8) - [github.com/psf/black: 23.10.1 → 23.12.0](https://github.com/psf/black/compare/23.10.1...23.12.0) - [github.com/codespell-project/codespell: v2.2.5 → v2.2.6](https://github.com/codespell-project/codespell/compare/v2.2.5...v2.2.6) - [github.com/pre-commit/pre-commit-hooks: v4.4.0 → v4.5.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.4.0...v4.5.0) - [github.com/pre-commit/mirrors-mypy: v1.3.0 → v1.7.1](https://github.com/pre-commit/mirrors-mypy/compare/v1.3.0...v1.7.1) * Attempt to fix ruff * Use isinstance --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- .pre-commit-config.yaml | 14 ++++++-------- zarr/core.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 029dcda58f..b4e7ab3ccf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,27 +6,25 @@ default_stages: [commit, push] default_language_version: python: python3 repos: - - repo: https://github.com/charliermarsh/ruff-pre-commit + - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.0.224' + rev: 'v0.1.8' hooks: - id: ruff - # Respect `exclude` and `extend-exclude` settings. - args: ["--force-exclude"] - repo: https://github.com/psf/black - rev: 23.10.1 + rev: 23.12.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell - rev: v2.2.5 + rev: v2.2.6 hooks: - id: codespell - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.3.0 + rev: v1.7.1 hooks: - id: mypy files: zarr diff --git a/zarr/core.py b/zarr/core.py index c07a31e95f..d22a9d79c3 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2536,7 +2536,7 @@ def hexdigest(self, hashname="sha1"): checksum = binascii.hexlify(self.digest(hashname=hashname)) # This is a bytes object on Python 3 and we want a str. - if type(checksum) is not str: + if not isinstance(checksum, str): checksum = checksum.decode("utf8") return checksum From e09ee149c4525213b07ace9eaf914ca9f552a703 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 10:01:20 -0700 Subject: [PATCH 0017/1043] chore: update pre-commit hooks (#1618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.8 → v0.1.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.8...v0.1.9) - [github.com/psf/black: 23.12.0 → 23.12.1](https://github.com/psf/black/compare/23.12.0...23.12.1) - [github.com/pre-commit/mirrors-mypy: v1.7.1 → v1.8.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.7.1...v1.8.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4e7ab3ccf..80d3439dc7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.8' + rev: 'v0.1.9' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 23.12.0 + rev: 23.12.1 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -24,7 +24,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.7.1 + rev: v1.8.0 hooks: - id: mypy files: zarr From cd139895b45a2d7d347c29b703aa2f6775a1e7c9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 10:24:59 -0700 Subject: [PATCH 0018/1043] Bump fsspec from 2023.12.1 to 2023.12.2 (#1606) * Bump fsspec from 2023.12.1 to 2023.12.2 Bumps [fsspec](https://github.com/fsspec/filesystem_spec) from 2023.12.1 to 2023.12.2. - [Commits](https://github.com/fsspec/filesystem_spec/compare/2023.12.1...2023.12.2) --- updated-dependencies: - dependency-name: fsspec dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Update requirements_dev_optional.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_optional.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 13385a243a..5916083cfc 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -18,6 +18,6 @@ pytest-cov==4.1.0 pytest-doctestplus==1.0.0 pytest-timeout==2.2.0 h5py==3.10.0 -fsspec==2023.12.1 -s3fs==2023.12.1 +fsspec==2023.12.2 +s3fs==2023.12.2 moto[server]>=4.0.8 From 5fb420fcfbabd484e663c78e55d04edd4ac9e486 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Dec 2023 11:32:02 +0100 Subject: [PATCH 0019/1043] Bump pytest-doctestplus from 1.0.0 to 1.1.0 (#1619) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.0.0 to 1.1.0. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.0.0...v1.1.0) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 5916083cfc..b4de5fd515 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -15,7 +15,7 @@ pymongo==4.6.1 # optional test requirements coverage pytest-cov==4.1.0 -pytest-doctestplus==1.0.0 +pytest-doctestplus==1.1.0 pytest-timeout==2.2.0 h5py==3.10.0 fsspec==2023.12.2 From 435a7ca7306fc31dc880ed23631e3af61bf53d66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 11:26:58 -0700 Subject: [PATCH 0020/1043] Bump pytest from 7.4.3 to 7.4.4 (#1622) --- requirements_dev_minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index afea816d87..94d3fff8a6 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,4 @@ numcodecs==0.12.1 msgpack-python==0.5.6 setuptools-scm==8.0.4 # test requirements -pytest==7.4.3 +pytest==7.4.4 From 6961fa9fb87ed73c85f979d84bfe65238933b5ae Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 20:31:31 -0800 Subject: [PATCH 0021/1043] chore: update pre-commit hooks (#1626) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.9 → v0.1.11](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.9...v0.1.11) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 80d3439dc7..340366ef53 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.9' + rev: 'v0.1.11' hooks: - id: ruff - repo: https://github.com/psf/black From ee518358d888caaabb6157c0498cb231d2ddb7a7 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 10 Jan 2024 06:47:36 -0800 Subject: [PATCH 0022/1043] Create TEAM.md (#1628) --- TEAM.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 TEAM.md diff --git a/TEAM.md b/TEAM.md new file mode 100644 index 0000000000..a95885ebe5 --- /dev/null +++ b/TEAM.md @@ -0,0 +1,25 @@ +## Active core-developers +- @joshmoore (Josh Moore) +- @jni (Juan Nunez-Iglesias) +- @rabernat (Ryan Abernathey) +- @jhamman (Joe Hamman) +- @d-v-b (Davis Bennett) +- @jakirkham (jakirkham) +- @martindurant (Martin Durant) + +## Emeritus core-developers +- @alimanfoo (Alistair Miles) +- @shoyer (Stephan Hoyer) +- @ryan-williams (Ryan Williams) +- @jrbourbeau (James Bourbeau) +- @mzjp2 (Zain Patel) +- @grlee77 (Gregory Lee) + +## Former core-developers +- @jeromekelleher (Jerome Kelleher) +- @tjcrone (Tim Crone) +- @funkey (Jan Funke) +- @shikharsg +- @Carreau (Matthias Bussonnier) +- @dazzag24 +- @WardF (Ward Fisher) From c7d66b4f8d7e9a4d50e5e01e5484ff8df612cb51 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Wed, 10 Jan 2024 20:52:42 +0100 Subject: [PATCH 0023/1043] Drop python 3.8 and numpy 1.20 (#1557) * Drop 3.8 and add 3.12 * Try removing line_profiler * Also bump the minimal numpy to 1.21 * Drop 3.12 again * Revert "Try removing line_profiler" This reverts commit 837854bec99a9d25aece2ead9666f01690d228cc. * Update release.rst --------- Co-authored-by: Joe Hamman Co-authored-by: jakirkham --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 8 ++++---- .github/workflows/releases.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- docs/release.rst | 3 +++ environment.yml | 2 +- pyproject.toml | 5 ++--- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ba05f23fcc..ec98af029e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.8.5, 3.9, 3.10, etc. + placeholder: 3.9, 3.10, 3.11, etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 0c3c49d78d..d74df9ce67 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,13 +15,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - numpy_version: ['>=1.22.0', '==1.20.*'] + python-version: ['3.9', '3.10', '3.11'] + numpy_version: ['>=1.22.0', '==1.21.*'] exclude: - python-version: '3.10' - numpy_version: '==1.20.*' + numpy_version: '==1.21.*' - python-version: '3.11' - numpy_version: '==1.20.*' + numpy_version: '==1.21.*' services: redis: image: redis diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 8d8512294d..31a7e2770c 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.0.0 name: Install Python with: - python-version: '3.8' + python-version: '3.9' - name: Install PyBuild run: | diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index eeee5b704d..5c3252c0ba 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: True matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v4 with: diff --git a/docs/release.rst b/docs/release.rst index c18e0b8c20..a3e0831ba4 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -49,6 +49,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Drop Python 3.8 and NumPy 1.20 + By :user:`Josh Moore `; :issue:`1557`. + * Cache result of ``FSStore._fsspec_installed()``. By :user:`Janick Martinez Esturo ` :issue:`1581`. diff --git a/environment.yml b/environment.yml index dc99507427..ff2f9eedef 100644 --- a/environment.yml +++ b/environment.yml @@ -4,7 +4,7 @@ channels: dependencies: - wheel - numcodecs >= 0.6.4 - - numpy >= 1.20 + - numpy >= 1.21 - pip - pip: - asciitree diff --git a/pyproject.toml b/pyproject.toml index 33e8573830..a85e49e82c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,10 +10,10 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ 'asciitree', - 'numpy>=1.20,!=1.21.0', + 'numpy>=1.21.1', 'fasteners', 'numcodecs>=0.10.0', ] @@ -30,7 +30,6 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', From 6ad464bb04bffa83b9665dd09caf0f8aaf6b367d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 10 Jan 2024 11:59:46 -0800 Subject: [PATCH 0024/1043] Add Norman Rzepka to core-dev team (#1630) --- TEAM.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TEAM.md b/TEAM.md index a95885ebe5..6a22d83d1f 100644 --- a/TEAM.md +++ b/TEAM.md @@ -6,6 +6,7 @@ - @d-v-b (Davis Bennett) - @jakirkham (jakirkham) - @martindurant (Martin Durant) +- @normanrz (Norman Rzepka) ## Emeritus core-developers - @alimanfoo (Alistair Miles) From a292dc43f8d0181214ded83124ebd4f85db0ff50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 11:14:02 -0800 Subject: [PATCH 0025/1043] chore: update pre-commit hooks (#1633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.11 → v0.1.13](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.11...v0.1.13) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 340366ef53..7d1f9254ae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.11' + rev: 'v0.1.13' hooks: - id: ruff - repo: https://github.com/psf/black From 68c87bb51d922487647fa6188392caf8c1d9a83c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Jan 2024 20:33:42 +0100 Subject: [PATCH 0026/1043] Bump actions/download-artifact from 3 to 4 (#1611) * Bump actions/download-artifact from 3 to 4 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Also bump upload-artifact see https://github.com/actions/download-artifact?tab=readme-ov-file#breaking-changes > Downloading artifacts that were created from action/upload-artifact@v3 and below are not supported. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman Co-authored-by: Josh Moore --- .github/workflows/releases.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 31a7e2770c..250c6112c8 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -36,7 +36,7 @@ jobs: else echo "All seem good" fi - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: releases path: dist @@ -45,7 +45,7 @@ jobs: needs: [build_artifacts] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: releases path: dist @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: releases path: dist From 1d56da0eb54f64840b1fb0f42c72622233f2f1f6 Mon Sep 17 00:00:00 2001 From: Jeff Peck Date: Tue, 16 Jan 2024 07:00:17 -0500 Subject: [PATCH 0027/1043] Update tutorial.rst to include section about accessing Zip Files on S3 (#1615) * Update tutorial.rst to include section about accessing Zip Files on S3 Per discussion here, add information about about accessing zip files on s3: https://github.com/zarr-developers/zarr-python/discussions/1613 * Update release.rst * Implement d-v-b's suggestions --------- Co-authored-by: Davis Bennett Co-authored-by: Josh Moore --- docs/release.rst | 2 ++ docs/tutorial.rst | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index a3e0831ba4..ab74a3debd 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -45,6 +45,8 @@ Docs * Minor tweak to advanced indexing tutorial examples. By :user:`Ross Barnowski ` :issue:`1550`. +* Added section about accessing zip files that are on s3. + By :user:`Jeff Peck ` :issue:`1613`. Maintenance ~~~~~~~~~~~ diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 4099bac1c8..351eef064a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -1000,6 +1000,31 @@ separately from Zarr. .. _tutorial_copy: +Accessing Zip Files on S3 +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The built-in `ZipStore` will only work with paths on the local file-system, however +it is also possible to access ``.zarr.zip`` data on the cloud. Here is an example of +accessing a zipped Zarr file on s3: + + >>> s3_path = "s3://path/to/my.zarr.zip" + >>> + >>> s3 = s3fs.S3FileSystem() + >>> f = s3.open(s3_path) + >>> fs = ZipFileSystem(f, mode="r") + >>> store = FSMap("", fs, check=False) + >>> + >>> # cache is optional, but may be a good idea depending on the situation + >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) + >>> z = zarr.group(store=cache) + +This store can also be generated with ``fsspec``'s handler chaining, like so: + + >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") + +This can be especially useful if you have a very large ``.zarr.zip`` file on s3 +and only need to access a small portion of it. + Consolidating metadata ~~~~~~~~~~~~~~~~~~~~~~ From 8ac8553f25eb338d6044d1232b4a643036979486 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 16 Jan 2024 09:14:10 -0800 Subject: [PATCH 0028/1043] doc(v3): add v3 roadmap and design document (#1583) * doc(v3): add v3 roadmap and design document * Update v3-roadmap-and-design.md * updates after latest round of reviews * Update v3-roadmap-and-design.md Co-authored-by: Norman Rzepka * Update v3-roadmap-and-design.md Co-authored-by: Sanket Verma --------- Co-authored-by: Norman Rzepka Co-authored-by: Sanket Verma --- v3-roadmap-and-design.md | 429 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 429 insertions(+) create mode 100644 v3-roadmap-and-design.md diff --git a/v3-roadmap-and-design.md b/v3-roadmap-and-design.md new file mode 100644 index 0000000000..696799e56f --- /dev/null +++ b/v3-roadmap-and-design.md @@ -0,0 +1,429 @@ +# Zarr Python Roadmap + +- Status: draft +- Author: Joe Hamman +- Created On: October 31, 2023 +- Input from: + - Davis Bennett / @d-v-b + - Norman Rzepka / @normanrz + - Deepak Cherian @dcherian + - Brian Davis / @monodeldiablo + - Oliver McCormack / @olimcc + - Ryan Abernathey / @rabernat + - Jack Kelly / @JackKelly + - Martin Durrant / @martindurant + +## Introduction + +This document lays out a design proposal for version 3.0 of the [Zarr-Python](https://zarr.readthedocs.io/en/stable/) package. A specific focus of the design is to bring Zarr-Python's API up to date with the [Zarr V3 specification](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html), with the hope of enabling the development of the many features and extensions that motivated the V3 Spec. The ideas presented here are expected to result in a major release of Zarr-Python (version 3.0) including significant a number of breaking API changes. +For clarity, “V3” will be used to describe the version of the Zarr specification and “3.0” will be used to describe the release tag of the Zarr-Python project. + +### Current status of V3 in Zarr-Python + +During the development of the V3 Specification, a [prototype implementation](https://github.com/zarr-developers/zarr-python/pull/898) was added to the Zarr-Python library. Since that implementation, the V3 spec evolved in significant ways and as a result, the Zarr-Python library is now out of sync with the approved spec. Downstream libraries (e.g. [Xarray](https://github.com/pydata/xarray)) have added support for this implementation and will need to migrate to the accepted spec when its available in Zarr-Python. + +## Goals + +- Provide a complete implementation of Zarr V3 through the Zarr-Python API +- Clear the way for exciting extensions / ZEPs (i.e. [sharding](https://zarr-specs.readthedocs.io/en/latest/v3/codecs/sharding-indexed/v1.0.html), [variable chunking](https://zarr.dev/zeps/draft/ZEP0003.html), etc.) +- Provide a developer API that can be used to implement and register V3 extensions +- Improve the performance of Zarr-Python by streamlining the interface between the Store layer and higher level APIs (e.g. Groups and Arrays) +- Clean up the internal and user facing APIs +- Improve code quality and robustness (e.g. achieve 100% type hint coverage) +- Align the Zarr-Python array API with the [array API Standard](https://data-apis.org/array-api/latest/) + +## Examples of what 3.0 will enable? +1. Reading and writing V3 spec-compliant groups and arrays +2. V3 extensions including sharding and variable chunking. +3. Improved performance by leveraging concurrency when creating/reading/writing to stores (imagine a `create_hierarchy(zarr_objects)` function). +4. User-developed extensions (e.g. storage-transformers) can be registered with Zarr-Python at runtime + +## Non-goals (of this document) + +- Implementation of any unaccepted Zarr V3 extensions +- Major revisions to the Zarr V3 spec + +## Requirements + +1. Read and write spec compliant V2 and V3 data +2. Limit unnecessary traffic to/from the store +3. Cleanly define the Array/Group/Store abstractions +4. Cleanly define how V2 will be supported going forward +5. Provide a clear roadmap to help users upgrade to 3.0 +6. Developer tools / hooks for registering extensions + +## Design + +### Async API + +Zarr-Python is an IO library. As such, supporting concurrent action against the storage layer is critical to achieving acceptable performance. The Zarr-Python 2 was not designed with asynchronous computation in mind and as a result has struggled to effectively leverage the benefits of concurrency. At one point, `getitems` and `setitems` support was added to the Zarr store model but that is only used for operating on a set of chunks in a single variable. + +With Zarr-Python 3.0, we have the opportunity to revisit this design. The proposal here is as follows: + +1. The `Store` interface will be entirely async. +2. On top of the async `Store` interface, we will provide an `AsyncArray` and `AsyncGroup` interface. +3. Finally, the primary user facing API will be synchronous `Array` and `Group` classes that wrap the async equivalents. + +**Examples** + +- **Store** + + ```python + class Store: + ... + async def get(self, key: str) -> bytes: + ... + async def get_partial_values(self, key_ranges: List[Tuple[str, Tuple[int, Optional[int]]]]) -> bytes: + ... + # (no sync interface here) + ``` +- **Array** + + ```python + class AsyncArray: + ... + + async def getitem(self, selection: Selection) -> np.ndarray: + # the core logic for getitem goes here + + class Array: + _async_array: AsyncArray + + def __getitem__(self, selection: Selection) -> np.ndarray: + return sync(self._async_array.getitem(selection)) + ``` +- **Group** + + ```python + class AsyncGroup: + ... + + async def create_group(self, path: str, **kwargs) -> AsyncGroup: + # the core logic for create_group goes here + + class Group: + _async_group: AsyncGroup + + def create_group(self, path: str, **kwargs) -> Group: + return sync(self._async_group.create_group(path, **kwargs)) + ``` +**Internal Synchronization API** + +With the `Store` and core `AsyncArray`/ `AsyncGroup` classes being predominantly async, Zarr-Python will need an internal API to provide a synchronous API. The proposal here is to use the approach in [fsspec](https://github.com/fsspec/filesystem_spec/blob/master/fsspec/asyn.py) to provide a high-level `sync` function that takes an `awaitable` and runs it in its managed IO Loop / thread. + +**FAQ** +1. Why two levels of Arrays/groups? + a. First, this is an intentional decision and departure from the current Zarrita implementation + b. The idea is that users rarely want to mix interfaces. Either they are working within an async context (currently quite rare) or they are in a typical synchronous context. + c. Splitting the two will allow us to clearly define behavior on the `AsyncObj` and simply wrap it in the `SyncObj`. +2. What if a store is only has a synchronous backend? + a. First off, this is expected to be a fairly rare occurrence. Most storage backends have async interfaces. + b. But in the event a storage backend doesn’t have a async interface, there is nothing wrong with putting synchronous code in `async` methods. There are approaches to enabling concurrent action through wrappers like AsyncIO's `loop.run_in_executor` ([ref 1](https://stackoverflow.com/questions/38865050/is-await-in-python3-cooperative-multitasking ), [ref 2](https://stackoverflow.com/a/43263397/732596), [ref 3](https://bbc.github.io/cloudfit-public-docs/asyncio/asyncio-part-5.html), [ref 4](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor). +3. Will Zarr help manage the async contexts encouraged by some libraries (e.g. [AioBotoCore](https://aiobotocore.readthedocs.io/en/latest/tutorial.html#using-botocore))? + a. Many async IO libraries require entering an async context before interacting with the API. We expect some experimentation to be needed here but the initial design will follow something close to what fsspec does ([example in s3fs](https://github.com/fsspec/s3fs/blob/949442693ec940b35cda3420c17a864fbe426567/s3fs/core.py#L527)). +4. Why not provide a synchronous Store interface? + a. We could but this design is simpler. It would mean supporting it in the `AsyncGroup` and `AsyncArray` classes which, may be more trouble than its worth. Storage backends that do not have an async API will be encouraged to wrap blocking calls in an async wrapper (e.g. `loop.run_in_executor`). + +### Store API + +The `Store` API is specified directly in the V3 specification. All V3 stores should implement this abstract API, omitting Write and List support as needed. As described above, all stores will be expected to expose the required methods as async methods. + +**Example** + +```python +class ReadWriteStore: + ... + async def get(self, key: str) -> bytes: + ... + + async def get_partial_values(self, key_ranges: List[Tuple[str, int, int]) -> bytes: + ... + + async def set(self, key: str, value: Union[bytes, bytearray, memoryview]) -> None: + ... # required for writable stores + + async def set_partial_values(self, key_start_values: List[Tuple[str, int, Union[bytes, bytearray, memoryview]]]) -> None: + ... # required for writable stores + + async def list(self) -> List[str]: + ... # required for listable stores + + async def list_prefix(self, prefix: str) -> List[str]: + ... # required for listable stores + + async def list_dir(self, prefix: str) -> List[str]: + ... # required for listable stores + + # additional (optional methods) + async def getsize(self, prefix: str) -> int: + ... + + async def rename(self, src: str, dest: str) -> None + ... + +``` + +Recognizing that there are many Zarr applications today that rely on the `MutableMapping` interface supported by Zarr-Python 2, a wrapper store will be developed to allow existing stores to plug directly into this API. + +### Array API + +The user facing array interface will implement a subset of the [Array API Standard](https://data-apis.org/array-api/latest/). Most of the computational parts of the Array API Standard don’t fit into Zarr right now. That’s okay. What matters most is that we ensure we can give downstream applications a compliant API. + +*Note, Zarr already does most of this so this is more about formalizing the relationship than a substantial change in API.* + +| | Included | Not Included | Unknown / Maybe possible? | +| --- | --- | --- | --- | +| Attributes | `dtype` | `mT` | `device` | +| | `ndim` | `T` | | +| | `shape` | | | +| | `size` | | | +| Methods | `__getitem__` | `__array_namespace__` | `to_device` | +| | `__setitem__` | `__abs__` | `__bool__` | +| | `__eq__` | `__add__` | `__complex__` | +| | `__bool__` | `__and__` | `__dlpack__` | +| | | `__floordiv__` | `__dlpack_device__` | +| | | `__ge__` | `__float__` | +| | | `__gt__` | `__index__` | +| | | `__invert__` | `__int__` | +| | | `__le__` | | +| | | `__lshift__` | | +| | | `__lt__` | | +| | | `__matmul__` | | +| | | `__mod__` | | +| | | `__mul__` | | +| | | `__ne__` | | +| | | `__neg__` | | +| | | `__or__` | | +| | | `__pos__` | | +| | | `__pow__` | | +| | | `__rshift__` | | +| | | `__sub__` | | +| | | `__truediv__` | | +| | | `__xor__` | | +| Creation functions (`zarr.creation`) | `zeros` | | `arange` | +| | `zeros_like` | | `asarray` | +| | `ones` | | `eye` | +| | `ones_like` | | `from_dlpack` | +| | `full` | | `linspace` | +| | `full_like` | | `meshgrid` | +| | `empty` | | `tril` | +| | `empty_like` | | `triu` | + +In addition to the core array API defined above, the Array class should have the following Zarr specific properties: + +- `.metadata` (see Metadata Interface below) +- `.attrs` - (pull from metadata object) +- `.info` - (pull from existing property †) + +*† In Zarr-Python 2, the info property lists the store to identify initialized chunks. By default this will be turned off in 3.0 but will be configurable.* + +**Indexing** + +Zarr-Python currently supports `__getitem__` style indexing and the special `oindex` and `vindex` indexers. These are not part of the current Array API standard (see [data-apis/array-api\#669](https://github.com/data-apis/array-api/issues/669)) but they have been [proposed as a NEP](https://numpy.org/neps/nep-0021-advanced-indexing.html). Zarr-Python will maintain these in 3.0. + +We are also exploring a new high-level indexing API that will enabled optimized batch/concurrent loading of many chunks. We expect this to be important to enable performant loading of data in the context of sharding. See [this discussion](https://github.com/zarr-developers/zarr-python/discussions/1569) for more detail. + +Concurrent indexing across multiple arrays will be possible using the AsyncArray API. + +**Async and Sync Array APIs** + +Most the logic to support Zarr Arrays will live in the `AsyncArray` class. There are a few notable differences that should be called out. + +| Sync Method | Async Method | +| --- | --- | +| `__getitem__` | `getitem` | +| `__setitem__` | `setitem` | +| `__eq__` | `equals` | + +**Metadata interface** + +Zarr-Python 2.* closely mirrors the V2 spec metadata schema in the Array and Group classes. In 3.0, we plan to move the underlying metadata representation to a separate interface (e.g. `Array.metadata`). This interface will return either a `V2ArrayMetadata` or `V3ArrayMetadata` object (both will inherit from a parent `ArrayMetadataABC` class. The `V2ArrayMetadata` and `V3ArrayMetadata` classes will be responsible for producing valid JSON representations of their metadata, and yielding a consistent view to the `Array` or `Group` class. + +### Group API + +The main question is how closely we should follow the existing Zarr-Python implementation / `MutableMapping` interface. The table below shows the primary `Group` methods in Zarr-Python 2 and attempts to identify if and how they would be implemented in 3.0. + +| V2 Group Methods | `AsyncGroup` | `Group` | `h5py_compat.Group`` | +| --- | --- | --- | --- | +| `__len__` | `length` | `__len__` | `__len__` | +| `__iter__` | `__aiter__` | `__iter__` | `__iter__` | +| `__contains__` | `contains` | `__contains__` | `__contains__` | +| `__getitem__` | `getitem` | `__getitem__` | `__getitem__` | +| `__enter__` | N/A | N/A | `__enter__` | +| `__exit__` | N/A | N/A | `__exit__` | +| `group_keys` | `group_keys` | `group_keys` | N/A | +| `groups` | `groups` | `groups` | N/A | +| `array_keys` | `array_key` | `array_keys` | N/A | +| `arrays` | `arrays`* | `arrays` | N/A | +| `visit` | ? | ? | `visit` | +| `visitkeys` | ? | ? | ? | +| `visitvalues` | ? | ? | ? | +| `visititems` | ? | ? | `visititems` | +| `tree` | `tree` | `tree` | `Both` | +| `create_group` | `create_group` | `create_group` | `create_group` | +| `require_group` | N/A | N/A | `require_group` | +| `create_groups` | ? | ? | N/A | +| `require_groups` | ? | ? | ? | +| `create_dataset` | N/A | N/A | `create_dataset` | +| `require_dataset` | N/A | N/A | `require_dataset` | +| `create` | `create_array` | `create_array` | N/A | +| `empty` | `empty` | `empty` | N/A | +| `zeros` | `zeros` | `zeros` | N/A | +| `ones` | `ones` | `ones` | N/A | +| `full` | `full` | `full` | N/A | +| `array` | `create_array` | `create_array` | N/A | +| `empty_like` | `empty_like` | `empty_like` | N/A | +| `zeros_like` | `zeros_like` | `zeros_like` | N/A | +| `ones_like` | `ones_like` | `ones_like` | N/A | +| `full_like` | `full_like` | `full_like` | N/A | +| `move` | `move` | `move` | `move` | + +**`zarr.h5compat.Group`** + +Zarr-Python 2.* made an attempt to align its API with that of [h5py](https://docs.h5py.org/en/stable/index.html). With 3.0, we will relax this alignment in favor of providing an explicit compatibility module (`zarr.h5py_compat`). This module will expose the `Group` and `Dataset` APIs that map to Zarr-Python’s `Group` and `Array` objects. + +### Creation API + +Zarr-Python 2.* bundles together the creation and serialization of Zarr objects. Zarr-Python 3.* will make it possible to create objects in memory separate from serializing them. This will specifically enable writing hierarchies of Zarr objects in a single batch step. For example: + +```python + +arr1 = Array(shape=(10, 10), path="foo/bar", dtype="i4", store=store) +arr2 = Array(shape=(10, 10), path="foo/spam", dtype="f8", store=store) + +arr1.save() +arr2.save() + +# or equivalently + +zarr.save_many([arr1 ,arr2]) +``` + +*Note: this batch creation API likely needs additional design effort prior to implementation.* + +### Plugin API + +Zarr V3 was designed to be extensible at multiple layers. Zarr-Python will support these extensions through a combination of [Abstract Base Classes](https://docs.python.org/3/library/abc.html) (ABCs) and [Entrypoints](https://packaging.python.org/en/latest/specifications/entry-points/). + +**ABCs** + +Zarr V3 will expose Abstract base classes for the following objects: + +- `Store`, `ReadStore`, `ReadWriteStore`, `ReadListStore`, and `ReadWriteListStore` +- `BaseArray`, `SynchronousArray`, and `AsynchronousArray` +- `BaseGroup`, `SynchronousGroup`, and `AsynchronousGroup` +- `Codec`, `ArrayArrayCodec`, `ArrayBytesCodec`, `BytesBytesCodec` + +**Entrypoints** + +Lots more thinking here but the idea here is to provide entrypoints for `data type`, `chunk grid`, `chunk key encoding`, `codecs`, `storage_transformers` and `stores`. These might look something like: + +``` +entry_points=""" + [zarr.codecs] + blosc_codec=codec_plugin:make_blosc_codec + zlib_codec=codec_plugin:make_zlib_codec +""" +``` + +### Python type hints and static analysis + +Target 100% Mypy coverage in 3.0 source. + +### Observability + +A persistent problem in Zarr-Python is diagnosing problems that span many parts of the stack. To address this in 3.0, we will add a basic logging framework that can be used to debug behavior at various levels of the stack. We propose to add the separate loggers for the following namespaces: + +- `array` +- `group` +- `store` +- `codec` + +These should be documented such that users know how to activate them and developers know how to use them when developing extensions. + +### Dependencies + +Today, Zarr-Python has the following required dependencies: + +```python +dependencies = [ + 'asciitree', + 'numpy>=1.20,!=1.21.0', + 'fasteners', + 'numcodecs>=0.10.0', +] +``` + +What other dependencies should be considered? + +1. Attrs - Zarrita makes extensive use of the Attrs library +2. Fsspec - Zarrita has a hard dependency on Fsspec. This could be easily relaxed though. + +## Breaking changes relative to Zarr-Python 2.* + +1. H5py compat moved to a stand alone module? +2. `Group.__getitem__` support moved to `Group.members.__getitem__`? +3. Others? + +## Open questions + +1. How to treat V2 + a. Note: Zarrita currently implements a separate `V2Array` and `V3Array` classes. This feels less than ideal. + b. We could easily convert metadata from v2 to the V3 Array, but what about writing? + c. Ideally, we don’t have completely separate code paths. But if its too complicated to support both within one interface, its probably better. +2. How and when to remove the current implementation of V3. + a. It's hidden behind a hard-to-use feature flag so we probably don't need to do anything. +4. How to model runtime configuration? +5. Which extensions belong in Zarr-Python and which belong in separate packages? + a. We don't need to take a strong position on this here. It's likely that someone will want to put Sharding in. That will be useful to develop in parallel because it will give us a good test case for the plugin interface. + +## Testing + +Zarr-python 3.0 adds a major new dimension to Zarr: Async support. This also comes with a compatibility risk, we will need to thoroughly test support in key execution environments. Testing plan: +- Reuse the existing test suite for testing the `v3` API. + - `xfail` tests that expose breaking changes with `3.0 - breaking change` description. This will help identify additional and/or unintentional breaking changes + - Rework tests that were only testing internal APIs. +- Add a set of functional / integration tests targeting real-world workflows in various contexts (e.g. w/ Dask) + +## Development process + +Zarr-Python 3.0 will introduce a number of new APIs and breaking changes to existing APIs. In order to facilitate ongoing support for Zarr-Python 2.*, we will take on the following development process: + +- Create a `v3` branch that can be use for developing the core functionality apart from the `main` branch. This will allow us to support ongoing work and bug fixes on the `main` branch. +- Put the `3.0` APIs inside a `zarr.v3` module. Imports from this namespace will all be new APIs that users can develop and test against once the `v3` branch is merged to `main`. +- Kickstart the process by pulling in the current state of `zarrita` - which has many of the features described in this design. +- Release a series of 2.* releases with the `v3` namespace +- When `v3` is complete, move contents of `v3` to the package root + +**Milestones** + +Below are a set of specific milestones leading toward the completion of this process. As work begins, we expect this list to grow in specificity. + +1. Port current version of Zarrita to Zarr-Python +2. Formalize Async interface by splitting `Array` and `Group` objects into Sync and Async versions +4. Implement "fancy" indexing operations on the `AsyncArray` +6. Implement an abstract base class for the `Store` interface and a wrapper `Store` to make use of existing `MutableMapping` stores. +7. Rework the existing unit test suite to use the `v3` namespace. +8. Develop a plugin interface for extensions +9. Develop a set of functional and integration tests +10. Work with downstream libraries (Xarray, Dask, etc.) to test new APIs + +## TODOs + +The following subjects are not covered in detail above but perhaps should be. Including them here so they are not forgotten. + +1. [Store] Should Zarr provide an API for caching objects after first read/list/etc. Read only stores? +2. [Array] buffer protocol support +3. [Array] `meta_array` support +4. [Extensions] Define how Zarr-Python will consume the various plugin types +5. [Misc] H5py compatibility requires a bit more work and a champion to drive it forward. +6. [Misc] Define `chunk_store` API in 3.0 +7. [Misc] Define `synchronizer` API in 3.0 + +## References + +1. [Zarr-Python repository](https://github.com/zarr-developers/zarr-python) +2. [Zarr core specification (version 3.0) — Zarr specs documentation](https://zarr-specs.readthedocs.io/en/latest/v3/core/v3.0.html#) +3. [Zarrita repository](https://github.com/scalableminds/zarrita) +4. [Async-Zarr](https://github.com/martindurant/async-zarr) +5. [Zarr-Python Discussion Topic](https://github.com/zarr-developers/zarr-python/discussions/1569) From a81db0782535ba04c32c277102a6457d118a73e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 11:50:32 -0800 Subject: [PATCH 0029/1043] chore: update pre-commit hooks (#1636) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.13 → v0.1.14](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.13...v0.1.14) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7d1f9254ae..a7f48d7cd6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.13' + rev: 'v0.1.14' hooks: - id: ruff - repo: https://github.com/psf/black From 4f2ace4b8708cf91f4bd29ae3ed210a3e66f235c Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Wed, 14 Feb 2024 04:09:41 -0800 Subject: [PATCH 0030/1043] Fix zarr sync (#1663) This patch removes fasteners and disables zarr.sync which uses process and thread Co-authored-by: Wei Ouyang --- pyproject.toml | 2 +- zarr/sync.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a85e49e82c..4da3079808 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ requires-python = ">=3.9" dependencies = [ 'asciitree', 'numpy>=1.21.1', - 'fasteners', + 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] dynamic = [ diff --git a/zarr/sync.py b/zarr/sync.py index 2e843f6557..03046a4a32 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -3,8 +3,6 @@ from threading import Lock from typing import Protocol -import fasteners - class Synchronizer(Protocol): """Base class for synchronizers.""" @@ -49,6 +47,8 @@ def __init__(self, path): self.path = path def __getitem__(self, item): + import fasteners + path = os.path.join(self.path, item) lock = fasteners.InterProcessLock(path) return lock From 0b0ac8857a52653fdb500cc9da7b51b0ec8a05b5 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Thu, 15 Feb 2024 01:47:27 +0530 Subject: [PATCH 0031/1043] Update release.rst (#1621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update release.rst * Update release.rst * Change 2.16.2 → 2.17.0 * Update moto for test_s3 * Skip bsddb3 tests to prevent warning failure * Fix more user warning tests * Fix even more user warning tests * Skip coverage for importorskips * Move to have_X skip method for deps * Update release.rst (PR#1663) * Fix test_core.py 'compile' issues * Add black formatting * Drop Windows/3.9 build due to unrelated failures * fix typo --------- Co-authored-by: Davis Bennett Co-authored-by: Josh Moore --- .github/workflows/windows-testing.yml | 2 +- docs/release.rst | 40 +++++++++++++++++++++++++++ requirements_dev_optional.txt | 2 +- zarr/tests/test_core.py | 28 +++++++++++++------ zarr/tests/test_storage.py | 2 +- zarr/tests/util.py | 24 ++++++++++++++++ 6 files changed, 87 insertions(+), 11 deletions(-) diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 5c3252c0ba..0ef7f21758 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: True matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: ['3.10', '3.11'] steps: - uses: actions/checkout@v4 with: diff --git a/docs/release.rst b/docs/release.rst index ab74a3debd..0f199aadd2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,12 +18,20 @@ Release notes Unreleased ---------- +.. _release_2.17.0: + +2.17.0 +------ + Enhancements ~~~~~~~~~~~~ * Added type hints to ``zarr.creation.create()``. By :user:`David Stansby ` :issue:`1536`. +* Pyodide support: Don't require fasteners on Emscripten. + By :user:`Hood Chatham ` :issue:`1663`. + Docs ~~~~ @@ -45,9 +53,21 @@ Docs * Minor tweak to advanced indexing tutorial examples. By :user:`Ross Barnowski ` :issue:`1550`. +* Automatically document array members using sphinx-automodapi. + By :user:`David Stansby ` :issue:`1547`. + +* Add a markdown file documenting the current and former core-developer team. + By :user:`Joe Hamman ` :issue:`1628`. + +* Add Norman Rzepka to core-dev team. + By :user:`Joe Hamman ` :issue:`1630`. + * Added section about accessing zip files that are on s3. By :user:`Jeff Peck ` :issue:`1613`. +* Add V3 roadmap and design document. + By :user:`Joe Hamman ` :issue:`1583`. + Maintenance ~~~~~~~~~~~ @@ -75,6 +95,26 @@ Maintenance * Remove ``sphinx-rtd-theme`` dependency from ``pyproject.toml``. By :user:`Sanket Verma ` :issue:`1563`. +* Remove ``CODE_OF_CONDUCT.md`` file from the Zarr-Python repository. + By :user:`Sanket Verma ` :issue:`1572`. + +* Bump version of black in pre-commit. + By :user:`David Stansby ` :issue:`1559`. + +* Use list comprehension where applicable. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1555`. + +* Use format specification mini-language to format string. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1558`. + +* Single startswith() call instead of multiple ones. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1556`. + +* Move codespell options around. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1196`. + +* Remove unused mypy ignore comments. + By :user:`David Stansby ` :issue:`1602`. .. _release_2.16.1: diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index b4de5fd515..d1ee5a891d 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -20,4 +20,4 @@ pytest-timeout==2.2.0 h5py==3.10.0 fsspec==2023.12.2 s3fs==2023.12.2 -moto[server]>=4.0.8 +moto[server]>=5.0.1 diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index a3fde4050d..cf15703497 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -73,7 +73,15 @@ ) from zarr.tests.test_storage_v3 import DummyStorageTransfomer from zarr.util import buffer_size -from zarr.tests.util import abs_container, skip_test_env_var, have_fsspec, mktemp +from zarr.tests.util import ( + abs_container, + have_bsddb3, + have_fsspec, + have_lmdb, + have_sqlite3, + mktemp, + skip_test_env_var, +) from zarr.types import DIMENSION_SEPARATOR # noinspection PyMethodMayBeStatic @@ -2038,9 +2046,11 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3") class TestArrayWithDBMStoreBerkeleyDB(TestArray): def create_store(self): - bsddb3 = pytest.importorskip("bsddb3") + import bsddb3 + path = mktemp(suffix=".dbm") atexit.register(os.remove, path) store = DBMStore(path, flag="n", open=bsddb3.btopen) @@ -2050,9 +2060,9 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStore(TestArray): def create_store(self): - pytest.importorskip("lmdb") path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) store = LMDBStore(path, buffers=True) @@ -2065,9 +2075,9 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStoreNoBuffers(TestArray): def create_store(self): - pytest.importorskip("lmdb") path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) store = LMDBStore(path, buffers=False) @@ -2077,9 +2087,9 @@ def test_nbytes_stored(self): pass # not implemented +@pytest.mark.skipif(have_sqlite3 is False, reason="needs sqlite3") class TestArrayWithSQLiteStore(TestArray): def create_store(self): - pytest.importorskip("sqlite3") path = mktemp(suffix=".db") atexit.register(atexit_rmtree, path) store = SQLiteStore(path) @@ -2758,9 +2768,11 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") +@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3") class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayV3): def create_store(self) -> DBMStoreV3: - bsddb3 = pytest.importorskip("bsddb3") + import bsddb3 + path = mktemp(suffix=".dbm") atexit.register(os.remove, path) store = DBMStoreV3(path, flag="n", open=bsddb3.btopen) @@ -2771,11 +2783,11 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") +@pytest.mark.skipif(have_lmdb is False, reason="needs lmdb") class TestArrayWithLMDBStoreV3(TestArrayV3): lmdb_buffers = True def create_store(self) -> LMDBStoreV3: - pytest.importorskip("lmdb") path = mktemp(suffix=".lmdb") atexit.register(atexit_rmtree, path) store = LMDBStoreV3(path, buffers=self.lmdb_buffers) @@ -2797,9 +2809,9 @@ def test_nbytes_stored(self): @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled") +@pytest.mark.skipif(have_sqlite3 is False, reason="needs sqlite3") class TestArrayWithSQLiteStoreV3(TestArrayV3): def create_store(self): - pytest.importorskip("sqlite3") path = mktemp(suffix=".db") atexit.register(atexit_rmtree, path) store = SQLiteStoreV3(path) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 25863749d8..e4e3d93f5f 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1396,7 +1396,7 @@ def s3(request): port = 5555 endpoint_uri = "http://127.0.0.1:%d/" % port proc = subprocess.Popen( - shlex.split("moto_server s3 -p %d" % port), + shlex.split("moto_server -p %d" % port), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL, ) diff --git a/zarr/tests/util.py b/zarr/tests/util.py index b4f00f703d..b3c3249cab 100644 --- a/zarr/tests/util.py +++ b/zarr/tests/util.py @@ -69,6 +69,30 @@ def skip_test_env_var(name): have_fsspec = False +try: + import bsddb3 # noqa: F401 + + have_bsddb3 = True +except ImportError: # pragma: no cover + have_bsddb3 = False + + +try: + import lmdb # noqa: F401 + + have_lmdb = True +except ImportError: # pragma: no cover + have_lmdb = False + + +try: + import sqlite3 # noqa: F401 + + have_sqlite3 = True +except ImportError: # pragma: no cover + have_sqlite3 = False + + def abs_container(): from azure.core.exceptions import ResourceExistsError import azure.storage.blob as asb From e50b47196eb4e4071158baba22567713ad012837 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:04:51 +0100 Subject: [PATCH 0032/1043] Bump numpy from 1.24.3 to 1.26.1 (#1543) Bumps [numpy](https://github.com/numpy/numpy) from 1.24.3 to 1.26.1. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.24.3...v1.26.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Josh Moore Co-authored-by: Joe Hamman --- requirements_dev_numpy.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index a6135bd831..c8c5f7d7ab 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.24.3 +numpy==1.26.1 From 81bbb2e7f28d64335d835523057041f11cdc7843 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:05:20 +0100 Subject: [PATCH 0033/1043] chore: update pre-commit hooks (#1642) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pre-commit hooks updates: - [github.com/astral-sh/ruff-pre-commit: v0.1.14 → v0.2.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.1.14...v0.2.1) - [github.com/psf/black: 23.12.1 → 24.2.0](https://github.com/psf/black/compare/23.12.1...24.2.0) * run black incl. comments for '...' --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- .pre-commit-config.yaml | 4 ++-- zarr/convenience.py | 1 + zarr/core.py | 8 +++++--- zarr/indexing.py | 10 +++++----- zarr/n5.py | 1 + zarr/storage.py | 1 + zarr/sync.py | 1 + zarr/types.py | 1 + 8 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7f48d7cd6..c7d4f32c68 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.1.14' + rev: 'v0.2.1' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 23.12.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell diff --git a/zarr/convenience.py b/zarr/convenience.py index 9c0deeea47..b4b8bb5293 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -1,4 +1,5 @@ """Convenience functions for storing and loading data.""" + import itertools import os import re diff --git a/zarr/core.py b/zarr/core.py index d22a9d79c3..5727afa884 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2060,9 +2060,11 @@ def _process_chunk( index_selection = PartialChunkIterator(chunk_selection, self.chunks) for start, nitems, partial_out_selection in index_selection: expected_shape = [ - len(range(*partial_out_selection[i].indices(self.chunks[0] + 1))) - if i < len(partial_out_selection) - else dim + ( + len(range(*partial_out_selection[i].indices(self.chunks[0] + 1))) + if i < len(partial_out_selection) + else dim + ) for i, dim in enumerate(self.chunks) ] if isinstance(cdata, UncompressedPartialReadBufferV3): diff --git a/zarr/indexing.py b/zarr/indexing.py index 3042147ebb..5a2b7c0eb4 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -545,11 +545,11 @@ def ix_(selection, shape): # replace slice and int as these are not supported by numpy.ix_ selection = [ - slice_to_range(dim_sel, dim_len) - if isinstance(dim_sel, slice) - else [dim_sel] - if is_integer(dim_sel) - else dim_sel + ( + slice_to_range(dim_sel, dim_len) + if isinstance(dim_sel, slice) + else [dim_sel] if is_integer(dim_sel) else dim_sel + ) for dim_sel, dim_len in zip(selection, shape) ] diff --git a/zarr/n5.py b/zarr/n5.py index 44b44e69e2..c50c18f718 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -1,5 +1,6 @@ """This module contains a storage class and codec to support the N5 format. """ + import os import struct import sys diff --git a/zarr/storage.py b/zarr/storage.py index aa27e98e6f..a26dc636db 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -14,6 +14,7 @@ path) and a `getsize` method (return the size in bytes of a given value). """ + import atexit import errno import glob diff --git a/zarr/sync.py b/zarr/sync.py index 03046a4a32..ba1c5df5b3 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -8,6 +8,7 @@ class Synchronizer(Protocol): """Base class for synchronizers.""" def __getitem__(self, item): + # see subclasses ... diff --git a/zarr/types.py b/zarr/types.py index 1de270f25c..cc29a350f5 100644 --- a/zarr/types.py +++ b/zarr/types.py @@ -10,4 +10,5 @@ class MetaArray(Protocol): def __array_function__(self, func, types, args, kwargs): + # To be extended ... From 367848836535e02eecd92a11ef734dd944285615 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:05:40 +0100 Subject: [PATCH 0034/1043] Bump ipywidgets from 8.1.0 to 8.1.1 (#1538) Bumps [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) from 8.1.0 to 8.1.1. - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.0...8.1.1) --- updated-dependencies: - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Davis Bennett Co-authored-by: Josh Moore --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index d1ee5a891d..0ac4922ce1 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -3,7 +3,7 @@ lmdb==1.4.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.0 +ipywidgets==8.1.1 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) From 720bea687b444b2082638eb7edc3bb6a4f8fa805 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Wed, 14 Feb 2024 22:14:43 +0100 Subject: [PATCH 0035/1043] Proper argument for numpy.reshape (#1425) `numpy.reshape` not only accepts a tuple of ints, but also a simple int. Besides `(10)` is not a tuple and is identical to `10`, unlike `(10,)`. --- zarr/tests/test_indexing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index af046e9d28..a3afc101c5 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -1632,7 +1632,7 @@ def test_set_selections_with_fields(): ), ( (slice(0, 10, 1),), - np.arange(0, 10).reshape((10)), + np.arange(0, 10).reshape(10), [(0, 10, (slice(0, 10, 1),))], ), ((0,), np.arange(0, 100).reshape((10, 10)), [(0, 10, (slice(0, 1, 1),))]), @@ -1644,7 +1644,7 @@ def test_set_selections_with_fields(): np.arange(0, 100).reshape((10, 10)), [(0, 1, (slice(0, 1, 1), slice(0, 1, 1)))], ), - ((0,), np.arange(0, 10).reshape((10)), [(0, 1, (slice(0, 1, 1),))]), + ((0,), np.arange(0, 10).reshape(10), [(0, 1, (slice(0, 1, 1),))]), pytest.param( (slice(5, 8, 1), slice(2, 4, 1), slice(0, 5, 1)), np.arange(2, 100002).reshape((10, 1, 10000)), From 74498538c180855172573f2983207f74674cbc1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Feb 2024 09:22:59 +0100 Subject: [PATCH 0036/1043] Bump ipywidgets from 8.1.1 to 8.1.2 (#1666) Bumps [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) from 8.1.1 to 8.1.2. - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.1...8.1.2) --- updated-dependencies: - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 0ac4922ce1..e94b814173 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -3,7 +3,7 @@ lmdb==1.4.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.1 +ipywidgets==8.1.2 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) From 3db41760e18fb0a69b5066e8c7aba9752a8c474e Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 15 Feb 2024 10:54:23 +0100 Subject: [PATCH 0037/1043] docs: ZIP-related tweaks (#1641) * docs: use 'ZIP archive' instead of 'zip file'; clarify utility of caching in s3 + ZIP example; style * docs: update release notes, correct spelling of greg lee's name in past release notes, and fix markup in past release notes * docs: use 'ZIP archive' instead of 'zip file'; clarify utility of caching in s3 + ZIP example; style * docs: update release notes, correct spelling of greg lee's name in past release notes, and fix markup in past release notes --- docs/release.rst | 20 ++++++++++---------- docs/tutorial.rst | 27 ++++++++++++++------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 0f199aadd2..b73dcec34f 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -62,8 +62,8 @@ Docs * Add Norman Rzepka to core-dev team. By :user:`Joe Hamman ` :issue:`1630`. -* Added section about accessing zip files that are on s3. - By :user:`Jeff Peck ` :issue:`1613`. +* Added section about accessing ZIP archives on s3. + By :user:`Jeff Peck ` :issue:`1613`, :issue:`1615`, and :user:`Davis Bennett ` :issue:`1641`. * Add V3 roadmap and design document. By :user:`Joe Hamman ` :issue:`1583`. @@ -157,10 +157,10 @@ Maintenance By :user:`Davis Bennett ` :issue:`1462`. * Style the codebase with ``ruff`` and ``black``. - By :user:`Davis Bennett` :issue:`1459` + By :user:`Davis Bennett ` :issue:`1459` * Ensure that chunks is tuple of ints upon array creation. - By :user:`Philipp Hanslovsky` :issue:`1461` + By :user:`Philipp Hanslovsky ` :issue:`1461` .. _release_2.15.0: @@ -548,7 +548,7 @@ Maintenance By :user:`Saransh Chopra ` :issue:`1079`. * Remove option to return None from _ensure_store. - By :user:`Greggory Lee ` :issue:`1068`. + By :user:`Gregory Lee ` :issue:`1068`. * Fix a typo of "integers". By :user:`Richard Scott ` :issue:`1056`. @@ -566,7 +566,7 @@ Enhancements Since the format is not yet finalized, the classes and functions are not automatically imported into the regular `zarr` name space. Setting the `ZARR_V3_EXPERIMENTAL_API` environment variable will activate them. - By :user:`Greggory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` + By :user:`Gregory Lee `; :issue:`898`, :issue:`1006`, and :issue:`1007` as well as by :user:`Josh Moore ` :issue:`1032`. * **Create FSStore from an existing fsspec filesystem**. If you have created @@ -688,7 +688,7 @@ Enhancements higher-level array creation and convenience functions still accept plain Python dicts or other mutable mappings for the ``store`` argument, but will internally convert these to a ``KVStore``. - By :user:`Greggory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. + By :user:`Gregory Lee `; :issue:`839`, :issue:`789`, and :issue:`950`. * Allow to assign array ``fill_values`` and update metadata accordingly. By :user:`Ryan Abernathey `, :issue:`662`. @@ -835,7 +835,7 @@ Bug fixes ~~~~~~~~~ * Fix FSStore.listdir behavior for nested directories. - By :user:`Greggory Lee `; :issue:`802`. + By :user:`Gregory Lee `; :issue:`802`. .. _release_2.9.4: @@ -919,7 +919,7 @@ Bug fixes By :user:`Josh Moore `; :issue:`781`. * avoid NumPy 1.21.0 due to https://github.com/numpy/numpy/issues/19325 - By :user:`Greggory Lee `; :issue:`791`. + By :user:`Gregory Lee `; :issue:`791`. Maintenance ~~~~~~~~~~~ @@ -931,7 +931,7 @@ Maintenance By :user:`Elliott Sales de Andrade `; :issue:`799`. * TST: add missing assert in test_hexdigest. - By :user:`Greggory Lee `; :issue:`801`. + By :user:`Gregory Lee `; :issue:`801`. .. _release_2.8.3: diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 351eef064a..1f7accab3a 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -774,7 +774,7 @@ the following code:: Any other compatible storage class could be used in place of :class:`zarr.storage.DirectoryStore` in the code examples above. For example, -here is an array stored directly into a Zip file, via the +here is an array stored directly into a ZIP archive, via the :class:`zarr.storage.ZipStore` class:: >>> store = zarr.ZipStore('data/example.zip', mode='w') @@ -798,12 +798,12 @@ Re-open and check that data have been written:: [42, 42, 42, ..., 42, 42, 42]], dtype=int32) >>> store.close() -Note that there are some limitations on how Zip files can be used, because items -within a Zip file cannot be updated in place. This means that data in the array +Note that there are some limitations on how ZIP archives can be used, because items +within a ZIP archive cannot be updated in place. This means that data in the array should only be written once and write operations should be aligned with chunk boundaries. Note also that the ``close()`` method must be called after writing any data to the store, otherwise essential records will not be written to the -underlying zip file. +underlying ZIP archive. Another storage alternative is the :class:`zarr.storage.DBMStore` class, added in Zarr version 2.2. This class allows any DBM-style database to be used for @@ -846,7 +846,7 @@ respectively require the `redis-py `_ and `pymongo `_ packages to be installed. For compatibility with the `N5 `_ data format, Zarr also provides -an N5 backend (this is currently an experimental feature). Similar to the zip storage class, an +an N5 backend (this is currently an experimental feature). Similar to the ZIP storage class, an :class:`zarr.n5.N5Store` can be instantiated directly:: >>> store = zarr.N5Store('data/example.n5') @@ -1000,12 +1000,13 @@ separately from Zarr. .. _tutorial_copy: -Accessing Zip Files on S3 -~~~~~~~~~~~~~~~~~~~~~~~~~ +Accessing ZIP archives on S3 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The built-in `ZipStore` will only work with paths on the local file-system, however -it is also possible to access ``.zarr.zip`` data on the cloud. Here is an example of -accessing a zipped Zarr file on s3: +The built-in :class:`zarr.storage.ZipStore` will only work with paths on the local file-system; however +it is possible to access ZIP-archived Zarr data on the cloud via the `ZipFileSystem `_ +class from ``fsspec``. The following example demonstrates how to access +a ZIP-archived Zarr group on s3 using `s3fs `_ and ``ZipFileSystem``: >>> s3_path = "s3://path/to/my.zarr.zip" >>> @@ -1014,7 +1015,7 @@ accessing a zipped Zarr file on s3: >>> fs = ZipFileSystem(f, mode="r") >>> store = FSMap("", fs, check=False) >>> - >>> # cache is optional, but may be a good idea depending on the situation + >>> # caching may improve performance when repeatedly reading the same data >>> cache = zarr.storage.LRUStoreCache(store, max_size=2**28) >>> z = zarr.group(store=cache) @@ -1022,7 +1023,7 @@ This store can also be generated with ``fsspec``'s handler chaining, like so: >>> store = zarr.storage.FSStore(url=f"zip::{s3_path}", mode="r") -This can be especially useful if you have a very large ``.zarr.zip`` file on s3 +This can be especially useful if you have a very large ZIP-archived Zarr array or group on s3 and only need to access a small portion of it. Consolidating metadata @@ -1161,7 +1162,7 @@ re-compression, and so should be faster. E.g.:: └── spam (100,) int64 >>> new_root['foo/bar/baz'][:] array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # zip stores need to be closed + >>> store2.close() # ZIP stores need to be closed .. _tutorial_strings: From d23683d21728d9be5a978719fbb75b1cb45b4441 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Feb 2024 08:37:26 +0100 Subject: [PATCH 0038/1043] Bump numpy from 1.26.1 to 1.26.4 (#1669) Bumps [numpy](https://github.com/numpy/numpy) from 1.26.1 to 1.26.4. - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.26.1...v1.26.4) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_numpy.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index c8c5f7d7ab..d8d6c3d097 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.26.1 +numpy==1.26.4 From 003ff33e70ce0a28411a7e9fde608354b1b8ee9b Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 16 Feb 2024 20:45:43 +0100 Subject: [PATCH 0039/1043] Change occurrences of % and format() to f-strings (#1423) Co-authored-by: Joe Hamman Co-authored-by: Josh Moore --- docs/release.rst | 3 + zarr/_storage/absstore.py | 6 +- zarr/_storage/store.py | 2 +- zarr/_storage/v3.py | 2 +- zarr/convenience.py | 44 +++++------ zarr/core.py | 20 ++--- zarr/creation.py | 4 +- zarr/errors.py | 4 +- zarr/hierarchy.py | 14 ++-- zarr/indexing.py | 37 +++++----- zarr/meta.py | 10 +-- zarr/meta_v1.py | 4 +- zarr/n5.py | 6 +- zarr/storage.py | 10 +-- zarr/tests/test_core.py | 10 +-- zarr/tests/test_meta.py | 146 ++++++++++++++----------------------- zarr/tests/test_storage.py | 6 +- zarr/util.py | 59 +++++++-------- 18 files changed, 167 insertions(+), 220 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index b73dcec34f..8ce4b2e33c 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,9 @@ Release notes Unreleased ---------- +* Change occurrences of % and format() to f-strings. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1423`. + .. _release_2.17.0: 2.17.0 diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index c9a113148c..b6b386f468 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -84,7 +84,7 @@ def __init__( blob_service_kwargs = blob_service_kwargs or {} client = ContainerClient( - "https://{}.blob.core.windows.net/".format(account_name), + f"https://{account_name}.blob.core.windows.net/", container, credential=account_key, **blob_service_kwargs, @@ -141,7 +141,7 @@ def __getitem__(self, key): try: return self.client.download_blob(blob_name).readall() except ResourceNotFoundError: - raise KeyError("Blob %s not found" % blob_name) + raise KeyError(f"Blob {blob_name} not found") def __setitem__(self, key, value): value = ensure_bytes(value) @@ -154,7 +154,7 @@ def __delitem__(self, key): try: self.client.delete_blob(self._append_path_to_prefix(key)) except ResourceNotFoundError: - raise KeyError("Blob %s not found" % key) + raise KeyError(f"Blob {key} not found") def __eq__(self, other): return ( diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 36b596769a..209f118534 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -227,7 +227,7 @@ def _validate_key(self, key: str): # TODO: Possibly allow key == ".zmetadata" too if we write a # consolidated metadata spec corresponding to this? ): - raise ValueError("keys starts with unexpected value: `{}`".format(key)) + raise ValueError(f"key starts with unexpected value: `{key}`") if key.endswith("/"): raise ValueError("keys may not end in /") diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 32e78f7a34..56bae74361 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -569,7 +569,7 @@ def __init__(self, store: StoreLike, metadata_key=meta_root + "consolidated/.zme consolidated_format = meta.get("zarr_consolidated_format", None) if consolidated_format != 1: raise MetadataError( - "unsupported zarr consolidated metadata format: %s" % consolidated_format + f"unsupported zarr consolidated metadata format: {consolidated_format}" ) # decode metadata diff --git a/zarr/convenience.py b/zarr/convenience.py index b4b8bb5293..7ca5d426f0 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -259,7 +259,7 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): try: grp = _create_group(_store, path=path, overwrite=True, zarr_version=zarr_version) for i, arr in enumerate(args): - k = "arr_{}".format(i) + k = f"arr_{i}" grp.create_dataset(k, data=arr, overwrite=True, zarr_version=zarr_version) for k, arr in kwargs.items(): grp.create_dataset(k, data=arr, overwrite=True, zarr_version=zarr_version) @@ -499,7 +499,7 @@ def __init__(self, log): self.log_file = log else: raise TypeError( - "log must be a callable function, file path or " "file-like object, found %r" % log + f"log must be a callable function, file path or file-like object, found {log!r}" ) def __enter__(self): @@ -526,9 +526,9 @@ def _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied): message = "dry run: " else: message = "all done: " - message += "{:,} copied, {:,} skipped".format(n_copied, n_skipped) + message += f"{n_copied:,} copied, {n_skipped:,} skipped" if not dry_run: - message += ", {:,} bytes copied".format(n_bytes_copied) + message += f", {n_bytes_copied:,} bytes copied" log(message) @@ -657,9 +657,7 @@ def copy_store( # check if_exists parameter valid_if_exists = ["raise", "replace", "skip"] if if_exists not in valid_if_exists: - raise ValueError( - "if_exists must be one of {!r}; found {!r}".format(valid_if_exists, if_exists) - ) + raise ValueError(f"if_exists must be one of {valid_if_exists!r}; found {if_exists!r}") # setup counting variables n_copied = n_skipped = n_bytes_copied = 0 @@ -720,20 +718,20 @@ def copy_store( if if_exists != "replace": if dest_key in dest: if if_exists == "raise": - raise CopyError("key {!r} exists in destination".format(dest_key)) + raise CopyError(f"key {dest_key!r} exists in destination") elif if_exists == "skip": do_copy = False # take action if do_copy: - log("copy {}".format(descr)) + log(f"copy {descr}") if not dry_run: data = source[source_key] n_bytes_copied += buffer_size(data) dest[dest_key] = data n_copied += 1 else: - log("skip {}".format(descr)) + log(f"skip {descr}") n_skipped += 1 # log a final message with a summary of what happened @@ -744,7 +742,7 @@ def copy_store( def _check_dest_is_group(dest): if not hasattr(dest, "create_dataset"): - raise ValueError("dest must be a group, got {!r}".format(dest)) + raise ValueError(f"dest must be a group, got {dest!r}") def copy( @@ -910,11 +908,9 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # check if_exists parameter valid_if_exists = ["raise", "replace", "skip", "skip_initialized"] if if_exists not in valid_if_exists: - raise ValueError( - "if_exists must be one of {!r}; found {!r}".format(valid_if_exists, if_exists) - ) + raise ValueError(f"if_exists must be one of {valid_if_exists!r}; found {if_exists!r}") if dest_h5py and if_exists == "skip_initialized": - raise ValueError("{!r} can only be used when copying to zarr".format(if_exists)) + raise ValueError(f"{if_exists!r} can only be used when copying to zarr") # determine name to copy to if name is None: @@ -934,9 +930,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ exists = dest is not None and name in dest if exists: if if_exists == "raise": - raise CopyError( - "an object {!r} already exists in destination " "{!r}".format(name, dest.name) - ) + raise CopyError(f"an object {name!r} already exists in destination {dest.name!r}") elif if_exists == "skip": do_copy = False elif if_exists == "skip_initialized": @@ -947,7 +941,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ # take action if do_copy: # log a message about what we're going to do - log("copy {} {} {}".format(source.name, source.shape, source.dtype)) + log(f"copy {source.name} {source.shape} {source.dtype}") if not dry_run: # clear the way @@ -1015,7 +1009,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ n_copied += 1 else: - log("skip {} {} {}".format(source.name, source.shape, source.dtype)) + log(f"skip {source.name} {source.shape} {source.dtype}") n_skipped += 1 elif root or not shallow: @@ -1026,16 +1020,14 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ exists_array = dest is not None and name in dest and hasattr(dest[name], "shape") if exists_array: if if_exists == "raise": - raise CopyError( - "an array {!r} already exists in destination " "{!r}".format(name, dest.name) - ) + raise CopyError(f"an array {name!r} already exists in destination {dest.name!r}") elif if_exists == "skip": do_copy = False # take action if do_copy: # log action - log("copy {}".format(source.name)) + log(f"copy {source.name}") if not dry_run: # clear the way @@ -1078,7 +1070,7 @@ def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_ n_copied += 1 else: - log("skip {}".format(source.name)) + log(f"skip {source.name}") n_skipped += 1 return n_copied, n_skipped, n_bytes_copied @@ -1327,7 +1319,7 @@ def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", ** store, storage_options=kwargs.get("storage_options"), mode=mode, zarr_version=zarr_version ) if mode not in {"r", "r+"}: - raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}".format(mode)) + raise ValueError(f"invalid mode, expected either 'r' or 'r+'; found {mode!r}") path = kwargs.pop("path", None) if store._store_version == 2: diff --git a/zarr/core.py b/zarr/core.py index 5727afa884..c3184c6652 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2396,11 +2396,11 @@ def _encode_chunk(self, chunk): def __repr__(self): t = type(self) - r = "<{}.{}".format(t.__module__, t.__name__) + r = f"<{t.__module__}.{t.__name__}" if self.name: - r += " %r" % self.name - r += " %s" % str(self.shape) - r += " %s" % self.dtype + r += f" {self.name!r}" + r += f" {str(self.shape)}" + r += f" {self.dtype}" if self._read_only: r += " read-only" r += ">" @@ -2436,11 +2436,11 @@ def info_items(self): def _info_items_nosync(self): def typestr(o): - return "{}.{}".format(type(o).__module__, type(o).__name__) + return f"{type(o).__module__}.{type(o).__name__}" def bytestr(n): if n > 2**10: - return "{} ({})".format(n, human_readable_size(n)) + return f"{n} ({human_readable_size(n)})" else: return str(n) @@ -2451,7 +2451,7 @@ def bytestr(n): items += [("Name", self.name)] items += [ ("Type", typestr(self)), - ("Data type", "%s" % self.dtype), + ("Data type", str(self.dtype)), ("Shape", str(self.shape)), ("Chunk shape", str(self.chunks)), ("Order", self.order), @@ -2461,7 +2461,7 @@ def bytestr(n): # filters if self.filters: for i, f in enumerate(self.filters): - items += [("Filter [%s]" % i, repr(f))] + items += [(f"Filter [{i}]", repr(f))] # compressor items += [("Compressor", repr(self.compressor))] @@ -2478,9 +2478,9 @@ def bytestr(n): if self.nbytes_stored > 0: items += [ ("No. bytes stored", bytestr(self.nbytes_stored)), - ("Storage ratio", "%.1f" % (self.nbytes / self.nbytes_stored)), + ("Storage ratio", f"{self.nbytes / self.nbytes_stored:.1f}"), ] - items += [("Chunks initialized", "{}/{}".format(self.nchunks_initialized, self.nchunks))] + items += [("Chunks initialized", f"{self.nchunks_initialized}/{self.nchunks}")] return items diff --git a/zarr/creation.py b/zarr/creation.py index d4f570895a..264715b040 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -287,7 +287,7 @@ def _kwargs_compat(compressor, fill_value, kwargs): compressor = compression else: - raise ValueError("bad value for compression: %r" % compression) + raise ValueError(f"bad value for compression: {compression!r}") # handle 'fillvalue' if "fillvalue" in kwargs: @@ -297,7 +297,7 @@ def _kwargs_compat(compressor, fill_value, kwargs): # ignore other keyword arguments for k in kwargs: - warn("ignoring keyword argument %r" % k) + warn(f"ignoring keyword argument {k!r}") return compressor, fill_value diff --git a/zarr/errors.py b/zarr/errors.py index 30c9b13d39..85789fbcbf 100644 --- a/zarr/errors.py +++ b/zarr/errors.py @@ -67,9 +67,7 @@ def __init__(self): def err_too_many_indices(selection, shape): - raise IndexError( - "too many indices for array; expected {}, got {}".format(len(shape), len(selection)) - ) + raise IndexError(f"too many indices for array; expected {len(shape)}, got {len(selection)}") class VindexInvalidSelectionError(_BaseZarrIndexError): diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 1cfea89c81..44af1d63d1 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -340,9 +340,9 @@ def __len__(self): def __repr__(self): t = type(self) - r = "<{}.{}".format(t.__module__, t.__name__) + r = f"<{t.__module__}.{t.__name__}" if self.name: - r += " %r" % self.name + r += f" {self.name!r}" if self._read_only: r += " read-only" r += ">" @@ -358,7 +358,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def info_items(self): def typestr(o): - return "{}.{}".format(type(o).__module__, type(o).__name__) + return f"{type(o).__module__}.{type(o).__name__}" items = [] @@ -1157,17 +1157,15 @@ def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, **kwargs shape = normalize_shape(shape) if shape != a.shape: raise TypeError( - "shape do not match existing array; expected {}, got {}".format(a.shape, shape) + f"shape do not match existing array; expected {a.shape}, got {shape}" ) dtype = np.dtype(dtype) if exact: if dtype != a.dtype: - raise TypeError( - "dtypes do not match exactly; expected {}, got {}".format(a.dtype, dtype) - ) + raise TypeError(f"dtypes do not match exactly; expected {a.dtype}, got {dtype}") else: if not np.can_cast(dtype, a.dtype): - raise TypeError("dtypes ({}, {}) cannot be safely cast".format(dtype, a.dtype)) + raise TypeError(f"dtypes ({dtype}, {a.dtype}) cannot be safely cast") return a else: diff --git a/zarr/indexing.py b/zarr/indexing.py index 5a2b7c0eb4..9889fcadad 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -338,8 +338,8 @@ def __init__(self, selection, array): else: raise IndexError( - "unsupported selection item for basic indexing; " - "expected integer or slice, got {!r}".format(type(dim_sel)) + f"unsupported selection item for basic indexing; " + f"expected integer or slice, got {type(dim_sel)!r}" ) dim_indexers.append(dim_indexer) @@ -370,8 +370,8 @@ def __init__(self, dim_sel, dim_len, dim_chunk_len): # check shape if dim_sel.shape[0] != dim_len: raise IndexError( - "Boolean array has the wrong length for dimension; " - "expected {}, got {}".format(dim_len, dim_sel.shape[0]) + f"Boolean array has the wrong length for dimension; " + f"expected {dim_len}, got { dim_sel.shape[0]}" ) # store attributes @@ -610,9 +610,9 @@ def __init__(self, selection, array): else: raise IndexError( - "unsupported selection item for orthogonal indexing; " - "expected integer, slice, integer array or Boolean " - "array, got {!r}".format(type(dim_sel)) + f"unsupported selection item for orthogonal indexing; " + f"expected integer, slice, integer array or Boolean " + f"array, got {type(dim_sel)!r}" ) dim_indexers.append(dim_indexer) @@ -698,8 +698,8 @@ def __init__(self, selection, array): if dim_sel.step not in {1, None}: raise IndexError( - "unsupported selection item for block indexing; " - "expected integer or slice with step=1, got {!r}".format(type(dim_sel)) + f"unsupported selection item for block indexing; " + f"expected integer or slice with step=1, got {type(dim_sel)!r}" ) # Can't reuse wraparound_indices because it expects a numpy array @@ -715,8 +715,8 @@ def __init__(self, selection, array): else: raise IndexError( - "unsupported selection item for block indexing; " - "expected integer or slice, got {!r}".format(type(dim_sel)) + f"unsupported selection item for block indexing; " + f"expected integer or slice, got {type(dim_sel)!r}" ) dim_indexer = SliceDimIndexer(slice_, dim_len, dim_chunk_size) @@ -782,9 +782,9 @@ def __init__(self, selection, array): # validation if not is_coordinate_selection(selection, array): raise IndexError( - "invalid coordinate selection; expected one integer " - "(coordinate) array per dimension of the target array, " - "got {!r}".format(selection) + f"invalid coordinate selection; expected one integer " + f"(coordinate) array per dimension of the target array, " + f"got {selection!r}" ) # handle wraparound, boundscheck @@ -874,8 +874,8 @@ def __init__(self, selection, array): # validation if not is_mask_selection(selection, array): raise IndexError( - "invalid mask selection; expected one Boolean (mask)" - "array with the same shape as the target array, got {!r}".format(selection) + f"invalid mask selection; expected one Boolean (mask)" + f"array with the same shape as the target array, got {selection!r}" ) # convert to indices @@ -919,8 +919,7 @@ def check_fields(fields, dtype): # check type if not isinstance(fields, (str, list, tuple)): raise IndexError( - "'fields' argument must be a string or list of strings; found " - "{!r}".format(type(fields)) + f"'fields' argument must be a string or list of strings; found " f"{type(fields)!r}" ) if fields: if dtype.names is None: @@ -933,7 +932,7 @@ def check_fields(fields, dtype): # multiple field selection out_dtype = np.dtype([(f, dtype[f]) for f in fields]) except KeyError as e: - raise IndexError("invalid 'fields' argument, field not found: {!r}".format(e)) + raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") else: return out_dtype else: diff --git a/zarr/meta.py b/zarr/meta.py index d9797e4754..4b360270de 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -111,7 +111,7 @@ def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType # check metadata format zarr_format = meta.get("zarr_format", None) if zarr_format != cls.ZARR_FORMAT: - raise MetadataError("unsupported zarr format: %s" % zarr_format) + raise MetadataError(f"unsupported zarr format: {zarr_format}") # extract array metadata fields try: @@ -199,7 +199,7 @@ def decode_group_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType # check metadata format version zarr_format = meta.get("zarr_format", None) if zarr_format != cls.ZARR_FORMAT: - raise MetadataError("unsupported zarr format: %s" % zarr_format) + raise MetadataError(f"unsupported zarr format: {zarr_format}") meta = dict(zarr_format=zarr_format) return meta @@ -346,7 +346,7 @@ def decode_group_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType # # check metadata format version # zarr_format = meta.get("zarr_format", None) # if zarr_format != cls.ZARR_FORMAT: - # raise MetadataError("unsupported zarr format: %s" % zarr_format) + # raise MetadataError(f"unsupported zarr format: {zarr_format}") assert "attributes" in meta # meta = dict(attributes=meta['attributes']) @@ -383,7 +383,7 @@ def decode_hierarchy_metadata(cls, s: Union[MappingType, bytes, str]) -> Mapping # check metadata format # zarr_format = meta.get("zarr_format", None) # if zarr_format != "https://purl.org/zarr/spec/protocol/core/3.0": - # raise MetadataError("unsupported zarr format: %s" % zarr_format) + # raise MetadataError(f"unsupported zarr format: {zarr_format}") if set(meta.keys()) != { "zarr_format", "metadata_encoding", @@ -518,7 +518,7 @@ def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType meta["storage_transformers"] = storage_transformers except Exception as e: - raise MetadataError("error decoding metadata: %s" % e) + raise MetadataError(f"error decoding metadata: {e}") else: return meta diff --git a/zarr/meta_v1.py b/zarr/meta_v1.py index 4ac381f2ca..65bfd3488e 100644 --- a/zarr/meta_v1.py +++ b/zarr/meta_v1.py @@ -10,7 +10,7 @@ def decode_metadata(b): meta = json.loads(s) zarr_format = meta.get("zarr_format", None) if zarr_format != 1: - raise MetadataError("unsupported zarr format: %s" % zarr_format) + raise MetadataError(f"unsupported zarr format: {zarr_format}") try: meta = dict( zarr_format=meta["zarr_format"], @@ -23,7 +23,7 @@ def decode_metadata(b): order=meta["order"], ) except Exception as e: - raise MetadataError("error decoding metadata: %s" % e) + raise MetadataError(f"error decoding metadata: {e}") else: return meta diff --git a/zarr/n5.py b/zarr/n5.py index c50c18f718..fdd3d5babf 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -826,9 +826,9 @@ def decode(self, chunk, out=None) -> bytes: if out is not None: # out should only be used if we read a complete chunk - assert chunk_shape == self.chunk_shape, "Expected chunk of shape {}, found {}".format( - self.chunk_shape, chunk_shape - ) + assert ( + chunk_shape == self.chunk_shape + ), f"Expected chunk of shape {self.chunk_shape}, found {chunk_shape}" if self._compressor: self._compressor.decode(chunk, out) diff --git a/zarr/storage.py b/zarr/storage.py index a26dc636db..73a6dc9630 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -2700,14 +2700,12 @@ def listdir(self, path=None): path = normalize_storage_path(path) sep = "_" if path == "" else "/" keys = self.cursor.execute( - """ + f""" SELECT DISTINCT SUBSTR(m, 0, INSTR(m, "/")) AS l FROM ( SELECT LTRIM(SUBSTR(k, LENGTH(?) + 1), "/") || "/" AS m FROM zarr WHERE k LIKE (? || "{sep}%") ) ORDER BY l ASC - """.format( - sep=sep - ), + """, (path, path), ) keys = list(map(operator.itemgetter(0), keys)) @@ -2863,7 +2861,7 @@ def __init__(self, prefix="zarr", dimension_separator=None, **kwargs): self.client = redis.Redis(**kwargs) def _key(self, key): - return "{prefix}:{key}".format(prefix=self._prefix, key=key) + return f"{self._prefix}:{key}" def __getitem__(self, key): return self.client[self._key(key)] @@ -2948,7 +2946,7 @@ def __init__(self, store: StoreLike, metadata_key=".zmetadata"): consolidated_format = meta.get("zarr_consolidated_format", None) if consolidated_format != 1: raise MetadataError( - "unsupported zarr consolidated metadata format: %s" % consolidated_format + f"unsupported zarr consolidated metadata format: {consolidated_format}" ) # decode metadata diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index cf15703497..d9447c0832 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -188,7 +188,7 @@ def test_store_has_text_keys(self): for k in z.chunk_store.keys(): if not isinstance(k, expected_type): # pragma: no cover - pytest.fail("Non-text key: %s" % repr(k)) + pytest.fail(f"Non-text key: {k!r}") z.store.close() @@ -202,7 +202,7 @@ def test_store_has_binary_values(self): try: ensure_ndarray(v) except TypeError: # pragma: no cover - pytest.fail("Non-bytes-like value: %s" % repr(v)) + pytest.fail(f"Non-bytes-like value: {v!r}") z.store.close() @@ -1212,7 +1212,7 @@ def test_dtypes(self): # datetime, timedelta for base_type in "Mm": for resolution in "D", "us", "ns": - dtype = "{}8[{}]".format(base_type, resolution) + dtype = f"{base_type}8[{resolution}]" z = self.create_array(shape=100, dtype=dtype, fill_value=0) assert z.dtype == np.dtype(dtype) a = np.random.randint( @@ -1402,7 +1402,7 @@ def compare_arrays(expected, actual, item_dtype): # convenience API for item_type in "int", " Tuple[np.dtype object_codec = codec_registry[codec_id](*args) except KeyError: # pragma: no cover raise ValueError( - "codec %r for object type %r is not " - "available; please provide an " - "object_codec manually" % (codec_id, key) + f"codec {codec_id!r} for object type {key!r} is not " + f"available; please provide an object_codec manually" ) return dtype, object_codec @@ -241,7 +240,7 @@ def is_total_slice(item, shape: Tuple[int]) -> bool: for it, sh in zip(item, shape) ) else: - raise TypeError("expected slice or tuple of slices, found %r" % item) + raise TypeError(f"expected slice or tuple of slices, found {item!r}") def normalize_resize_args(old_shape, *args): @@ -265,23 +264,23 @@ def normalize_resize_args(old_shape, *args): def human_readable_size(size) -> str: if size < 2**10: - return "%s" % size + return f"{size}" elif size < 2**20: - return "%.1fK" % (size / float(2**10)) + return f"{size / float(2**10):.1f}K" elif size < 2**30: - return "%.1fM" % (size / float(2**20)) + return f"{size / float(2**20):.1f}M" elif size < 2**40: - return "%.1fG" % (size / float(2**30)) + return f"{size / float(2**30):.1f}G" elif size < 2**50: - return "%.1fT" % (size / float(2**40)) + return f"{size / float(2**40):.1f}T" else: - return "%.1fP" % (size / float(2**50)) + return f"{size / float(2**50):.1f}P" def normalize_order(order: str) -> str: order = str(order).upper() if order not in ["C", "F"]: - raise ValueError("order must be either 'C' or 'F', found: %r" % order) + raise ValueError(f"order must be either 'C' or 'F', found: {order!r}") return order @@ -289,7 +288,7 @@ def normalize_dimension_separator(sep: Optional[str]) -> Optional[DIMENSION_SEPA if sep in (".", "/", None): return cast(Optional[DIMENSION_SEPARATOR], sep) else: - raise ValueError("dimension_separator must be either '.' or '/', found: %r" % sep) + raise ValueError(f"dimension_separator must be either '.' or '/', found: {sep!r}") def normalize_fill_value(fill_value, dtype: np.dtype): @@ -307,8 +306,8 @@ def normalize_fill_value(fill_value, dtype: np.dtype): if not isinstance(fill_value, str): raise ValueError( - "fill_value {!r} is not valid for dtype {}; must be a " - "unicode string".format(fill_value, dtype) + f"fill_value {fill_value!r} is not valid for dtype {dtype}; " + f"must be a unicode string" ) else: @@ -322,8 +321,8 @@ def normalize_fill_value(fill_value, dtype: np.dtype): except Exception as e: # re-raise with our own error message to be helpful raise ValueError( - "fill_value {!r} is not valid for dtype {}; nested " - "exception: {}".format(fill_value, dtype, e) + f"fill_value {fill_value!r} is not valid for dtype {dtype}; " + f"nested exception: {e}" ) return fill_value @@ -396,10 +395,10 @@ def info_html_report(items) -> str: report += "" for k, v in items: report += ( - "" - '%s' - '%s' - "" % (k, v) + f"" + f'{k}' + f'{v}' + f"" ) report += "" report += "" @@ -435,7 +434,7 @@ def get_children(self): def get_text(self): name = self.obj.name.split("/")[-1] or "/" if hasattr(self.obj, "shape"): - name += " {} {}".format(self.obj.shape, self.obj.dtype) + name += f" {self.obj.shape} {self.obj.dtype}" return name def get_type(self): @@ -463,7 +462,7 @@ def tree_get_icon(stype: str) -> str: elif stype == "Group": return tree_group_icon else: - raise ValueError("Unknown type: %s" % stype) + raise ValueError(f"Unknown type: {stype}") def tree_widget_sublist(node, root=False, expand=False): @@ -487,10 +486,10 @@ def tree_widget(group, expand, level): import ipytree except ImportError as error: raise ImportError( - "{}: Run `pip install zarr[jupyter]` or `conda install ipytree`" - "to get the required ipytree dependency for displaying the tree " - "widget. If using jupyterlab<3, you also need to run " - "`jupyter labextension install ipytree`".format(error) + f"{error}: Run `pip install zarr[jupyter]` or `conda install ipytree`" + f"to get the required ipytree dependency for displaying the tree " + f"widget. If using jupyterlab<3, you also need to run " + f"`jupyter labextension install ipytree`" ) result = ipytree.Tree() @@ -549,14 +548,10 @@ def _repr_mimebundle_(self, **kwargs): def check_array_shape(param, array, shape): if not hasattr(array, "shape"): - raise TypeError( - "parameter {!r}: expected an array-like object, got {!r}".format(param, type(array)) - ) + raise TypeError(f"parameter {param!r}: expected an array-like object, got {type(array)!r}") if array.shape != shape: raise ValueError( - "parameter {!r}: expected array with shape {!r}, got {!r}".format( - param, shape, array.shape - ) + f"parameter {param!r}: expected array with shape {shape!r}, got {array.shape!r}" ) From f80f697c2612cf41c5bdb158a602c1ae8a737e70 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:28:10 +0530 Subject: [PATCH 0040/1043] chore: update pre-commit hooks (#1672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.1 → v0.2.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.1...v0.2.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7d4f32c68..41b65f1d02 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.2.1' + rev: 'v0.2.2' hooks: - id: ruff - repo: https://github.com/psf/black From 54bc90c8682472cc40fba35ec6b313cb1f046c34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 09:57:06 -0800 Subject: [PATCH 0041/1043] Bump pymongo from 4.6.1 to 4.6.2 (#1674) Bumps [pymongo](https://github.com/mongodb/mongo-python-driver) from 4.6.1 to 4.6.2. - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/4.6.2/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.1...4.6.2) --- updated-dependencies: - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index e94b814173..85f6fccffc 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.16.0 # pyup: ignore redis==5.0.1 types-redis types-setuptools -pymongo==4.6.1 +pymongo==4.6.2 # optional test requirements coverage pytest-cov==4.1.0 From 70a15bbe595031ad24b82ca5cee9468a8229e775 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 10:09:11 -0800 Subject: [PATCH 0042/1043] Bump conda-incubator/setup-miniconda from 3.0.1 to 3.0.2 (#1677) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index 2cc0213781..d95b2bc540 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.1 + uses: conda-incubator/setup-miniconda@v3.0.2 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d74df9ce67..946b7efa7d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,7 +42,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.1 + uses: conda-incubator/setup-miniconda@v3.0.2 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 0ef7f21758..85e5c3e6b6 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.1 + - uses: conda-incubator/setup-miniconda@v3.0.2 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From ec4d2162828c2616a388dda2bdbcf40c8747a36d Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Tue, 27 Feb 2024 15:26:29 +0100 Subject: [PATCH 0043/1043] Update config.yml with Zulip --- .github/ISSUE_TEMPLATE/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 9cb5ec9a78..907121f858 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -3,8 +3,8 @@ contact_links: - name: ✨ Propose a new major feature url: https://github.com/zarr-developers/zarr-specs about: A new major feature should be discussed in the Zarr specifications repository. - - name: ❓ Discuss something on gitter - url: https://gitter.im/zarr-developers/community + - name: ❓ Discuss something on Zulip + url: https://ossci.zulipchat.com/ about: For questions like "How do I do X with Zarr?", you can move to our Gitter channel. - name: ❓ Discuss something on GitHub Discussions url: https://github.com/zarr-developers/zarr-python/discussions From a0e5559c38bf1a9d7c1a70a81f51f5eece5701c2 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Tue, 27 Feb 2024 17:04:50 +0100 Subject: [PATCH 0044/1043] Type dimension separator (#1620) Co-authored-by: Davis Bennett --- zarr/_storage/absstore.py | 5 ++- zarr/_storage/v3.py | 7 ++-- zarr/_storage/v3_storage_transformers.py | 3 +- zarr/creation.py | 2 +- zarr/storage.py | 45 +++++++++++++++++------- 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index b6b386f468..217b2a29e0 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -1,9 +1,12 @@ """This module contains storage classes related to Azure Blob Storage (ABS)""" +from typing import Optional import warnings + from numcodecs.compat import ensure_bytes from zarr.util import normalize_storage_path from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, Store, StoreV3 +from zarr.types import DIMENSION_SEPARATOR __doctest_requires__ = { ("ABSStore", "ABSStore.*"): ["azure.storage.blob"], @@ -67,7 +70,7 @@ def __init__( account_name=None, account_key=None, blob_service_kwargs=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, client=None, ): self._dimension_separator = dimension_separator diff --git a/zarr/_storage/v3.py b/zarr/_storage/v3.py index 56bae74361..4987f820cf 100644 --- a/zarr/_storage/v3.py +++ b/zarr/_storage/v3.py @@ -3,13 +3,14 @@ from collections import OrderedDict from collections.abc import MutableMapping from threading import Lock -from typing import Union, Dict, Any +from typing import Union, Dict, Any, Optional from zarr.errors import ( MetadataError, ReadOnlyError, ) from zarr.util import buffer_size, json_loads, normalize_storage_path +from zarr.types import DIMENSION_SEPARATOR from zarr._storage.absstore import ABSStoreV3 # noqa: F401 from zarr._storage.store import ( # noqa: F401 @@ -224,7 +225,9 @@ def get_partial_values(self, key_ranges): class MemoryStoreV3(MemoryStore, StoreV3): - def __init__(self, root=None, cls=dict, dimension_separator=None): + def __init__( + self, root=None, cls=dict, dimension_separator: Optional[DIMENSION_SEPARATOR] = None + ): if root is None: self.root = cls() else: diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index 3afc3823a3..37e56f8ecd 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -8,6 +8,7 @@ from zarr._storage.store import StorageTransformer, StoreV3, _rmdir_from_keys_v3 from zarr.util import normalize_storage_path +from zarr.types import DIMENSION_SEPARATOR MAX_UINT_64 = 2**64 - 1 @@ -118,7 +119,7 @@ def _copy_for_array(self, array, inner_store): return transformer_copy @property - def dimension_separator(self) -> str: + def dimension_separator(self) -> DIMENSION_SEPARATOR: assert ( self._dimension_separator is not None ), "dimension_separator is not initialized, first get a copy via _copy_for_array." diff --git a/zarr/creation.py b/zarr/creation.py index 264715b040..c541531d54 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -470,7 +470,7 @@ def open_array( write_empty_chunks=True, *, zarr_version=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, meta_array=None, **kwargs, ): diff --git a/zarr/storage.py b/zarr/storage.py index 73a6dc9630..f6903d29b2 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -41,7 +41,8 @@ from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like from numcodecs.registry import codec_registry from zarr.context import Context -from zarr.types import PathLike as Path +from zarr.types import PathLike as Path, DIMENSION_SEPARATOR +from zarr.util import NoLock from zarr.errors import ( MetadataError, @@ -327,7 +328,7 @@ def init_array( chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, storage_transformers=(), ): """Initialize an array store with the given configuration. Note that this is a low-level @@ -481,7 +482,7 @@ def _init_array_metadata( chunk_store: Optional[StoreLike] = None, filters=None, object_codec=None, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, storage_transformers=(), ): store_version = getattr(store, "_store_version", 2) @@ -1054,7 +1055,9 @@ class DirectoryStore(Store): """ - def __init__(self, path, normalize_keys=False, dimension_separator=None): + def __init__( + self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = None + ): # guard conditions path = os.path.abspath(path) if os.path.exists(path) and not os.path.isdir(path): @@ -1349,7 +1352,7 @@ def __init__( key_separator=None, mode="w", exceptions=(KeyError, PermissionError, IOError), - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, fs=None, check=False, create=False, @@ -1568,7 +1571,12 @@ class TempStore(DirectoryStore): # noinspection PyShadowingBuiltins def __init__( - self, suffix="", prefix="zarr", dir=None, normalize_keys=False, dimension_separator=None + self, + suffix="", + prefix="zarr", + dir=None, + normalize_keys=False, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, ): path = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) atexit.register(atexit_rmtree, path) @@ -1652,7 +1660,9 @@ class NestedDirectoryStore(DirectoryStore): """ - def __init__(self, path, normalize_keys=False, dimension_separator="/"): + def __init__( + self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = "/" + ): super().__init__(path, normalize_keys=normalize_keys) if dimension_separator is None: dimension_separator = "/" @@ -1765,7 +1775,7 @@ def __init__( compression=zipfile.ZIP_STORED, allowZip64=True, mode="a", - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, ): # store properties path = os.path.abspath(path) @@ -2058,7 +2068,7 @@ def __init__( mode=0o666, open=None, write_lock=True, - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **open_kwargs, ): if open is None: @@ -2073,6 +2083,7 @@ def __init__( self.mode = mode self.open = open self.write_lock = write_lock + self.write_mutex: Union[Lock, NoLock] if write_lock: # This may not be required as some dbm implementations manage their own # locks, but err on the side of caution. @@ -2229,7 +2240,13 @@ class LMDBStore(Store): """ - def __init__(self, path, buffers=True, dimension_separator=None, **kwargs): + def __init__( + self, + path, + buffers=True, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, + **kwargs, + ): import lmdb # set default memory map size to something larger than the lmdb default, which is @@ -2580,7 +2597,7 @@ class SQLiteStore(Store): >>> store.close() # don't forget to call this when you're done """ - def __init__(self, path, dimension_separator=None, **kwargs): + def __init__(self, path, dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs): import sqlite3 self._dimension_separator = dimension_separator @@ -2776,7 +2793,7 @@ def __init__( self, database="mongodb_zarr", collection="zarr_collection", - dimension_separator=None, + dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs, ): import pymongo @@ -2851,7 +2868,9 @@ class RedisStore(Store): """ - def __init__(self, prefix="zarr", dimension_separator=None, **kwargs): + def __init__( + self, prefix="zarr", dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs + ): import redis self._prefix = prefix From 99e03c684729b188457024a53afc45cb1b160027 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Wed, 28 Feb 2024 18:22:28 +0530 Subject: [PATCH 0045/1043] Replace Gitter with new Zulip Chat link (#1685) * Replace Gitter with Zulip * Replace Gitter with Zulip in remaining places --- .github/ISSUE_TEMPLATE/config.yml | 2 +- README.md | 6 +++--- docs/index.rst | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 907121f858..9ceaab2ae7 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -5,7 +5,7 @@ contact_links: about: A new major feature should be discussed in the Zarr specifications repository. - name: ❓ Discuss something on Zulip url: https://ossci.zulipchat.com/ - about: For questions like "How do I do X with Zarr?", you can move to our Gitter channel. + about: For questions like "How do I do X with Zarr?", you can move to our Zulip Chat. - name: ❓ Discuss something on GitHub Discussions url: https://github.com/zarr-developers/zarr-python/discussions about: For questions like "How do I do X with Zarr?", you can move to GitHub Discussions. diff --git a/README.md b/README.md index b035ffa597..e379c9719f 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,10 @@ - Gitter + Zulip - - + + diff --git a/docs/index.rst b/docs/index.rst index 06f79b7e7c..a5dbfbc5bf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,7 +25,7 @@ Zarr-Python `Installation `_ | `Source Repository `_ | `Issue Tracker `_ | -`Gitter `_ +`Zulip Chat `_ Zarr is a file storage format for chunked, compressed, N-dimensional arrays based on an open-source specification. From 67d5d82317451c9072a187efd6d638e718cdaced Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 02:03:35 +0530 Subject: [PATCH 0046/1043] Bump redis from 5.0.1 to 5.0.2 (#1688) Bumps [redis](https://github.com/redis/redis-py) from 5.0.1 to 5.0.2. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.1...v5.0.2) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 85f6fccffc..c3d747a47e 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,7 +8,7 @@ ipywidgets==8.1.2 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.1 +redis==5.0.2 types-redis types-setuptools pymongo==4.6.2 From 9c2a412d70ed717165966bc47615bdef195d68c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:48:15 +0100 Subject: [PATCH 0047/1043] Bump pypa/gh-action-pypi-publish from 1.8.11 to 1.8.12 (#1691) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.11 to 1.8.12. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.11...v1.8.12) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 250c6112c8..6d417042b5 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.11 + - uses: pypa/gh-action-pypi-publish@v1.8.12 with: user: __token__ password: ${{ secrets.pypi_password }} From 237f934f5ac7d7a04c6b144f97e54776eda628c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Mar 2024 01:02:48 +0530 Subject: [PATCH 0048/1043] Bump pytest-doctestplus from 1.1.0 to 1.2.0 (#1693) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.1.0 to 1.2.0. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.1.0...v1.2.0) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index c3d747a47e..0f4493b1d4 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -15,7 +15,7 @@ pymongo==4.6.2 # optional test requirements coverage pytest-cov==4.1.0 -pytest-doctestplus==1.1.0 +pytest-doctestplus==1.2.0 pytest-timeout==2.2.0 h5py==3.10.0 fsspec==2023.12.2 From 240bb824b86a68aa0eb6f03fecbe5439882607ca Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Wed, 6 Mar 2024 19:48:22 +0530 Subject: [PATCH 0049/1043] Fix RTD build (#1694) --- .readthedocs.yaml | 4 +++- docs/index.rst | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 08cac8d78d..e45cae1b45 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -16,4 +16,6 @@ python: extra_requirements: - docs -formats: all +formats: + - htmlzip + - pdf diff --git a/docs/index.rst b/docs/index.rst index a5dbfbc5bf..cf54e261af 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,7 +19,7 @@ Zarr-Python **Version**: |version| -**Download documentation**: `PDF/Zipped HTML/EPUB `_ +**Download documentation**: `PDF/Zipped HTML `_ **Useful links**: `Installation `_ | From a1fbedb18c1fc70f026c423fafca6d84ad88ce53 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Thu, 7 Mar 2024 00:53:47 +0530 Subject: [PATCH 0050/1043] Update release.rst for v2.17.1 (#1673) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update release.rst for v2.17.1 * Change the copyright year from 2023 → 2024. * Update release.rst for v2.17.1 --- LICENSE.txt | 2 +- docs/conf.py | 2 +- docs/release.rst | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 850a0d8772..a4de1c39d3 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015-2023 Zarr Developers +Copyright (c) 2015-2024 Zarr Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/docs/conf.py b/docs/conf.py index 318843a9fb..048e77f51d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,7 +72,7 @@ # General information about the project. project = "zarr" -copyright = "2023, Zarr Developers" +copyright = "2024, Zarr Developers" author = "Zarr Developers" version = zarr.__version__ diff --git a/docs/release.rst b/docs/release.rst index 8ce4b2e33c..037432ca58 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,9 +18,41 @@ Release notes Unreleased ---------- +.. _release_2.17.1: + +2.17.1 +------ + +Enhancements +~~~~~~~~~~~~ + * Change occurrences of % and format() to f-strings. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1423`. +* Proper argument for numpy.reshape. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1425`. + +* Add typing to dimension separator arguments. + By :user:`David Stansby ` :issue:`1620`. + +Docs +~~~~ + +* ZIP related tweaks. + By :user:`Davis Bennett ` :issue:`1641`. + +Maintenance +~~~~~~~~~~~ + +* Update config.yml with Zulip. + By :user:`Josh Moore `. + +* Replace Gitter with the new Zulip Chat link. + By :user:`Sanket Verma ` :issue:`1685`. + +* Fix RTD build. + By :user:`Sanket Verma ` :issue:`1694`. + .. _release_2.17.0: 2.17.0 From d986f8973eafbf847179d8c3f7e16451d0fcd63d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 8 Mar 2024 17:40:55 -0800 Subject: [PATCH 0051/1043] Bump pytest-timeout from 2.2.0 to 2.3.1 (#1697) Bumps [pytest-timeout](https://github.com/pytest-dev/pytest-timeout) from 2.2.0 to 2.3.1. - [Commits](https://github.com/pytest-dev/pytest-timeout/compare/2.2.0...2.3.1) --- updated-dependencies: - dependency-name: pytest-timeout dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 0f4493b1d4..b14381dd6e 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -16,7 +16,7 @@ pymongo==4.6.2 coverage pytest-cov==4.1.0 pytest-doctestplus==1.2.0 -pytest-timeout==2.2.0 +pytest-timeout==2.3.1 h5py==3.10.0 fsspec==2023.12.2 s3fs==2023.12.2 From d642da6320793c64dc227cf8062a9936d0fd398e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 10 Mar 2024 14:26:24 -0700 Subject: [PATCH 0052/1043] Bump conda-incubator/setup-miniconda from 3.0.2 to 3.0.3 (#1690) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.2...v3.0.3) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index d95b2bc540..dba6918514 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.2 + uses: conda-incubator/setup-miniconda@v3.0.3 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 946b7efa7d..fd2603ff95 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -42,7 +42,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.2 + uses: conda-incubator/setup-miniconda@v3.0.3 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index 85e5c3e6b6..d580ef3f0e 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.2 + - uses: conda-incubator/setup-miniconda@v3.0.3 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From 029cff71b86871cde76c7909cfecd28764953377 Mon Sep 17 00:00:00 2001 From: "Daniel Jahn (dahn)" Date: Sun, 10 Mar 2024 22:37:36 +0100 Subject: [PATCH 0053/1043] docs(tutorial.rst): fix link to GCSMap (#1689) --- docs/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1f7accab3a..214dd4f63f 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -868,7 +868,7 @@ implementations of the ``MutableMapping`` interface for Amazon S3 (`S3Map Distributed File System (`HDFSMap `_) and Google Cloud Storage (`GCSMap -`_), which +`_), which can be used with Zarr. Here is an example using S3Map to read an array created previously:: From 9fc4981ddfb0e032fcc76fa6585b5a66dc5d2f06 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Mon, 11 Mar 2024 03:19:28 +0530 Subject: [PATCH 0054/1043] Update installation.rst stating version support policy (#1665) * Update installation.rst stating version support policy * Update docs/installation.rst Co-authored-by: Joe Hamman * Update docs/installation.rst --------- Co-authored-by: Joe Hamman --- docs/installation.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 8553d451cb..35865c764d 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,6 +6,11 @@ Zarr depends on NumPy. It is generally best to `install NumPy appropriate for your operating system and Python distribution. Other dependencies should be installed automatically if using one of the installation methods below. +Note: Zarr has endorsed `Scientific-Python SPEC 0 `_ and now follows the version support window as outlined below: + +- Python: 36 months after initial release +- Core package dependencies (e.g. NumPy): 24 months after initial release + Install Zarr from PyPI:: $ pip install zarr From f58065b221452acd70235902ad59d920da6fb02f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 08:47:15 -0700 Subject: [PATCH 0055/1043] Bump pypa/gh-action-pypi-publish from 1.8.12 to 1.8.14 (#1700) Bumps [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish) from 1.8.12 to 1.8.14. - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.12...v1.8.14) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 6d417042b5..fe168d2862 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.12 + - uses: pypa/gh-action-pypi-publish@v1.8.14 with: user: __token__ password: ${{ secrets.pypi_password }} From bbac25472e0a781dc5c0256d26a481eacb27390b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 16:58:21 +0100 Subject: [PATCH 0056/1043] Bump pytest-doctestplus from 1.2.0 to 1.2.1 (#1699) Bumps [pytest-doctestplus](https://github.com/scientific-python/pytest-doctestplus) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/scientific-python/pytest-doctestplus/releases) - [Changelog](https://github.com/scientific-python/pytest-doctestplus/blob/main/CHANGES.rst) - [Commits](https://github.com/scientific-python/pytest-doctestplus/compare/v1.2.0...v1.2.1) --- updated-dependencies: - dependency-name: pytest-doctestplus dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index b14381dd6e..62b257ea70 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -15,7 +15,7 @@ pymongo==4.6.2 # optional test requirements coverage pytest-cov==4.1.0 -pytest-doctestplus==1.2.0 +pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 h5py==3.10.0 fsspec==2023.12.2 From 6fe553df925c224fcc0a12ecdd074997ce9e56f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:53:29 +0100 Subject: [PATCH 0057/1043] Bump redis from 5.0.2 to 5.0.3 (#1698) Bumps [redis](https://github.com/redis/redis-py) from 5.0.2 to 5.0.3. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.2...v5.0.3) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 62b257ea70..7ff673cebd 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,7 +8,7 @@ ipywidgets==8.1.2 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.2 +redis==5.0.3 types-redis types-setuptools pymongo==4.6.2 From f4f0b42d5ced9f777709fb87ff06ff09fbaa3055 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 25 Mar 2024 09:44:40 -0700 Subject: [PATCH 0058/1043] Add Python 3.12 to CI (#1719) * Update python-package.yml * bump numpy versions * bump min python version * Update release.rst --- .github/workflows/python-package.yml | 10 ++++++---- docs/release.rst | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fd2603ff95..2f9166ae96 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,13 +15,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11'] - numpy_version: ['>=1.22.0', '==1.21.*'] + python-version: ['3.9', '3.10', '3.11', '3.12'] + numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' - numpy_version: '==1.21.*' + numpy_version: '==1.23.*' - python-version: '3.11' - numpy_version: '==1.21.*' + numpy_version: '==1.23.*' + - python-version: '3.12' + numpy_version: '==1.23.*' services: redis: image: redis diff --git a/docs/release.rst b/docs/release.rst index 037432ca58..5c4da710b2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -53,6 +53,12 @@ Maintenance * Fix RTD build. By :user:`Sanket Verma ` :issue:`1694`. +* Add CI test environment for Python 3.12 + By :user:`Joe Hamman ` :issue:`1719`. + +* Bump minimum supported NumPy version to 1.23 (per spec 0000) + By :user:`Joe Hamman ` :issue:`1719`. + .. _release_2.17.0: 2.17.0 diff --git a/pyproject.toml b/pyproject.toml index 4da3079808..0be79f990e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ maintainers = [ requires-python = ">=3.9" dependencies = [ 'asciitree', - 'numpy>=1.21.1', + 'numpy>=1.23', 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] From 04e862cfcaf58aeb5e13e40b66007f44caa9bcae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 19:02:07 +0100 Subject: [PATCH 0059/1043] Bump pytest-cov from 4.1.0 to 5.0.0 (#1722) Bumps [pytest-cov](https://github.com/pytest-dev/pytest-cov) from 4.1.0 to 5.0.0. - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v4.1.0...v5.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 7ff673cebd..a3411acd67 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -14,7 +14,7 @@ types-setuptools pymongo==4.6.2 # optional test requirements coverage -pytest-cov==4.1.0 +pytest-cov==5.0.0 pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 h5py==3.10.0 From 0e58b79dd38911a80648f5a4cbf01439fe90c7a5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 00:27:27 +0530 Subject: [PATCH 0060/1043] chore: update pre-commit hooks (#1708) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.2.2 → v0.3.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.2.2...v0.3.3) - [github.com/psf/black: 24.2.0 → 24.3.0](https://github.com/psf/black/compare/24.2.0...24.3.0) - [github.com/pre-commit/mirrors-mypy: v1.8.0 → v1.9.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.8.0...v1.9.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sanket Verma --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 41b65f1d02..46aadb554b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.2.2' + rev: 'v0.3.3' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -24,7 +24,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.9.0 hooks: - id: mypy files: zarr From 8d0d910f27453157558d201e7a6a84bb565c87ad Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:46:34 -0700 Subject: [PATCH 0061/1043] chore: update pre-commit hooks (#1723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.3 → v0.3.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.3...v0.3.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 46aadb554b..6c2762f34d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.3' + rev: 'v0.3.4' hooks: - id: ruff - repo: https://github.com/psf/black From 2534413e2f8b56d9c64744419628a464d639f1dc Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 26 Mar 2024 18:44:50 -0700 Subject: [PATCH 0062/1043] Fix release notes (following #1719) (#1725) --- docs/release.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 5c4da710b2..116393d417 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,15 @@ Release notes Unreleased ---------- +Maintenance +~~~~~~~~~~~ + +* Add CI test environment for Python 3.12 + By :user:`Joe Hamman ` :issue:`1719`. + +* Bump minimum supported NumPy version to 1.23 (per spec 0000) + By :user:`Joe Hamman ` :issue:`1719`. + .. _release_2.17.1: 2.17.1 @@ -53,12 +62,6 @@ Maintenance * Fix RTD build. By :user:`Sanket Verma ` :issue:`1694`. -* Add CI test environment for Python 3.12 - By :user:`Joe Hamman ` :issue:`1719`. - -* Bump minimum supported NumPy version to 1.23 (per spec 0000) - By :user:`Joe Hamman ` :issue:`1719`. - .. _release_2.17.0: 2.17.0 From bad8dd0240861c48e768c265911aec2aec24481c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 27 Mar 2024 12:42:33 -0600 Subject: [PATCH 0063/1043] Override ipython repr methods. (#1724) Closes #1716 This avoids expensive lookups against object stores. --- docs/release.rst | 6 ++++++ zarr/hierarchy.py | 41 ++++++++++++++++++++++++++++++++++++ zarr/tests/test_hierarchy.py | 21 ++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 116393d417..fd48a53b38 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,12 @@ Release notes Unreleased ---------- +Enhancements +~~~~~~~~~~~~ + +* Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. + By :user:`Deepak Cherian ` :issue:`1716`. + Maintenance ~~~~~~~~~~~ diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 44af1d63d1..c88892c932 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -515,6 +515,13 @@ def _delitem_nosync(self, item): raise KeyError(item) def __getattr__(self, item): + # https://github.com/jupyter/notebook/issues/2014 + # Save a possibly expensive lookup (for e.g. against cloud stores) + # Note: The _ipython_display_ method is required to display the right info as a side-effect. + # It is simpler to pretend it doesn't exist. + if item in ["_ipython_canary_method_should_not_exist_", "_ipython_display_"]: + raise AttributeError + # allow access to group members via dot notation try: return self.__getitem__(item) @@ -1331,6 +1338,40 @@ def move(self, source, dest): self._write_op(self._move_nosync, source, dest) + # Override ipython repr methods, GH1716 + # https://ipython.readthedocs.io/en/stable/config/integrating.html#custom-methods + # " If the methods don’t exist, the standard repr() is used. If a method exists and + # returns None, it is treated the same as if it does not exist." + def _repr_html_(self): + return None + + def _repr_latex_(self): + return None + + def _repr_mimebundle_(self, **kwargs): + return None + + def _repr_svg_(self): + return None + + def _repr_png_(self): + return None + + def _repr_jpeg_(self): + return None + + def _repr_markdown_(self): + return None + + def _repr_javascript_(self): + return None + + def _repr_pdf_(self): + return None + + def _repr_json_(self): + return None + def _normalize_store_arg(store, *, storage_options=None, mode="r", zarr_version=None): if zarr_version is None: diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 6c08d7b88a..161e1eb813 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1,4 +1,5 @@ import atexit +import operator import os import sys import pickle @@ -87,6 +88,26 @@ def create_group( ) return g + def test_ipython_repr_methods(self): + g = self.create_group() + for method in [ + "html", + "json", + "javascript", + "markdown", + "svg", + "png", + "jpeg", + "latex", + "pdf", + "mimebundle", + ]: + assert operator.methodcaller(f"_repr_{method}_")(g) is None + with pytest.raises(AttributeError): + g._ipython_display_() + with pytest.raises(AttributeError): + g._ipython_canary_method_should_not_exist_() + def test_group_init_1(self): store, chunk_store = self.create_store() g = self.create_group(store, chunk_store=chunk_store) From 37e0a1a0c22f552daf6fd94fec5474a9b92db33d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:40:01 +0530 Subject: [PATCH 0064/1043] Bump pymongo from 4.6.2 to 4.6.3 (#1729) --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index a3411acd67..809d1c0eee 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -11,7 +11,7 @@ azure-storage-blob==12.16.0 # pyup: ignore redis==5.0.3 types-redis types-setuptools -pymongo==4.6.2 +pymongo==4.6.3 # optional test requirements coverage pytest-cov==5.0.0 From cb5b77a7fcf8312bc28d7970a31395c5690f5898 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Fri, 29 Mar 2024 05:37:50 +0530 Subject: [PATCH 0065/1043] Remove v1 and v2 specification (#1582) * Remove v1 and v2 specification * fix warning --------- Co-authored-by: Davis Bennett --- docs/release.rst | 4 +- docs/spec/v1.rst | 267 +--------------------- docs/spec/v2.rst | 562 +---------------------------------------------- docs/spec/v3.rst | 2 +- 4 files changed, 7 insertions(+), 828 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index fd48a53b38..9c75dc4feb 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1625,11 +1625,11 @@ Bug fixes Documentation ~~~~~~~~~~~~~ -* Some changes have been made to the :ref:`spec_v2` document to clarify +* Some changes have been made to the Zarr Specification v2 document to clarify ambiguities and add some missing information. These changes do not break compatibility with any of the material as previously implemented, and so the changes have been made in-place in the document without incrementing the document version number. See the - section on :ref:`spec_v2_changes` in the specification document for more information. + section on changes in the specification document for more information. * A new :ref:`tutorial_indexing` section has been added to the tutorial. * A new :ref:`tutorial_strings` section has been added to the tutorial (:issue:`135`, :issue:`175`). diff --git a/docs/spec/v1.rst b/docs/spec/v1.rst index 13f68ef36e..27a0490e0a 100644 --- a/docs/spec/v1.rst +++ b/docs/spec/v1.rst @@ -3,268 +3,5 @@ Zarr Storage Specification Version 1 ==================================== -This document provides a technical specification of the protocol and -format used for storing a Zarr array. The key words "MUST", "MUST -NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", -"RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be -interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is deprecated. See :ref:`spec` for the latest version. - -Storage -------- - -A Zarr array can be stored in any storage system that provides a -key/value interface, where a key is an ASCII string and a value is an -arbitrary sequence of bytes, and the supported operations are read -(get the sequence of bytes associated with a given key), write (set -the sequence of bytes associated with a given key) and delete (remove -a key/value pair). - -For example, a directory in a file system can provide this interface, -where keys are file names, values are file contents, and files can be -read, written or deleted via the operating system. Equally, an S3 -bucket can provide this interface, where keys are resource names, -values are resource contents, and resources can be read, written or -deleted via HTTP. - -Below an "array store" refers to any system implementing this -interface. - -Metadata --------- - -Each array requires essential configuration metadata to be stored, -enabling correct interpretation of the stored data. This metadata is -encoded using JSON and stored as the value of the 'meta' key within an -array store. - -The metadata resource is a JSON object. The following keys MUST be -present within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compression - A string identifying the primary compression library used to compress - each chunk of the array. -compression_opts - An integer, string or dictionary providing options to the primary - compression library. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array. -order - Either 'C' or 'F', defining the layout of bytes within each chunk of the - array. 'C' means row-major order, i.e., the last dimension varies fastest; - 'F' means column-major order, i.e., the first dimension varies fastest. - -Other keys MAY be present within the metadata object however they MUST -NOT alter the interpretation of the required fields defined above. - -For example, the JSON object below defines a 2-dimensional array of -64-bit little-endian floating point numbers with 10000 rows and 10000 -columns, divided into chunks of 1000 rows and 1000 columns (so there -will be 100 chunks in total arranged in a 10 by 10 grid). Within each -chunk the data are laid out in C contiguous order, and each chunk is -compressed using the Blosc compression library:: - - { - "chunks": [ - 1000, - 1000 - ], - "compression": "blosc", - "compression_opts": { - "clevel": 5, - "cname": "lz4", - "shuffle": 1 - }, - "dtype": "`_. The -format consists of 3 parts: a character describing the byteorder of -the data (``<``: little-endian, ``>``: big-endian, ``|``: -not-relevant), a character code giving the basic type of the array, -and an integer providing the number of bytes the type uses. The byte -order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data types. - -Structure data types (i.e., with multiple named fields) are encoded as -a list of two-element lists, following `NumPy array protocol type -descriptions (descr) -`_. -For example, the JSON list ``[["r", "|u1"], ["g", "|u1"], ["b", -"|u1"]]`` defines a data type composed of three single-byte unsigned -integers labelled 'r', 'g' and 'b'. - -Chunks ------- - -Each chunk of the array is compressed by passing the raw bytes for the -chunk through the primary compression library to obtain a new sequence -of bytes comprising the compressed chunk data. No header is added to -the compressed bytes or any other modification made. The internal -structure of the compressed bytes will depend on which primary -compressor was used. For example, the `Blosc compressor -`_ -produces a sequence of bytes that begins with a 16-byte header -followed by compressed data. - -The compressed sequence of bytes for each chunk is stored under a key -formed from the index of the chunk within the grid of chunks -representing the array. To form a string key for a chunk, the indices -are converted to strings and concatenated with the period character -('.') separating each index. For example, given an array with shape -(10000, 10000) and chunk shape (1000, 1000) there will be 100 chunks -laid out in a 10 by 10 grid. The chunk with indices (0, 0) provides -data for rows 0-999 and columns 0-999 and is stored under the key -'0.0'; the chunk with indices (2, 4) provides data for rows 2000-2999 -and columns 4000-4999 and is stored under the key '2.4'; etc. - -There is no need for all chunks to be present within an array -store. If a chunk is not present then it is considered to be in an -uninitialized state. An uninitialized chunk MUST be treated as if it -was uniformly filled with the value of the 'fill_value' field in the -array metadata. If the 'fill_value' field is ``null`` then the -contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of -any array dimension is not exactly divisible by the length of the -corresponding chunk dimension then some chunks will overhang the edge -of the array. The contents of any chunk region falling outside the -array are undefined. - -Attributes ----------- - -Each array can also be associated with custom attributes, which are -simple key/value items with application-specific meaning. Custom -attributes are encoded as a JSON object and stored under the 'attrs' -key within an array store. Even if the attributes are empty, the -'attrs' key MUST be present within an array store. - -For example, the JSON object below encodes three attributes named -'foo', 'bar' and 'baz':: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -Example -------- - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Initialize the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('example.zarr') - >>> zarr.init_store(store, shape=(20, 20), chunks=(10, 10), - ... dtype='i4', fill_value=42, compression='zlib', - ... compression_opts=1, overwrite=True) - -No chunks are initialized yet, so only the 'meta' and 'attrs' keys -have been set:: - - >>> import os - >>> sorted(os.listdir('example.zarr')) - ['attrs', 'meta'] - -Inspect the array metadata:: - - >>> print(open('example.zarr/meta').read()) - { - "chunks": [ - 10, - 10 - ], - "compression": "zlib", - "compression_opts": 1, - "dtype": ">> print(open('example.zarr/attrs').read()) - {} - -Set some data:: - - >>> z = zarr.Array(store) - >>> z[0:10, 0:10] = 1 - >>> sorted(os.listdir('example.zarr')) - ['0.0', 'attrs', 'meta'] - -Set some more data:: - - >>> z[0:10, 10:20] = 2 - >>> z[10:20, :] = 3 - >>> sorted(os.listdir('example.zarr')) - ['0.0', '0.1', '1.0', '1.1', 'attrs', 'meta'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> b = zlib.decompress(open('example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> a = np.frombuffer(b, dtype='>> a - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> z.attrs['foo'] = 42 - >>> z.attrs['bar'] = 'apples' - >>> z.attrs['baz'] = [1, 2, 3, 4] - >>> print(open('example.zarr/attrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } +The V1 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v2.rst b/docs/spec/v2.rst index c1e12e1218..deb6d46ce6 100644 --- a/docs/spec/v2.rst +++ b/docs/spec/v2.rst @@ -3,563 +3,5 @@ Zarr Storage Specification Version 2 ==================================== -This document provides a technical specification of the protocol and format -used for storing Zarr arrays. The key words "MUST", "MUST NOT", "REQUIRED", -"SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and -"OPTIONAL" in this document are to be interpreted as described in `RFC 2119 -`_. - -Status ------- - -This specification is the latest version. See :ref:`spec` for previous -versions. - -.. _spec_v2_storage: - -Storage -------- - -A Zarr array can be stored in any storage system that provides a key/value -interface, where a key is an ASCII string and a value is an arbitrary sequence -of bytes, and the supported operations are read (get the sequence of bytes -associated with a given key), write (set the sequence of bytes associated with -a given key) and delete (remove a key/value pair). - -For example, a directory in a file system can provide this interface, where -keys are file names, values are file contents, and files can be read, written -or deleted via the operating system. Equally, an S3 bucket can provide this -interface, where keys are resource names, values are resource contents, and -resources can be read, written or deleted via HTTP. - -Below an "array store" refers to any system implementing this interface. - -.. _spec_v2_array: - -Arrays ------- - -.. _spec_v2_array_metadata: - -Metadata -~~~~~~~~ - -Each array requires essential configuration metadata to be stored, enabling -correct interpretation of the stored data. This metadata is encoded using JSON -and stored as the value of the ".zarray" key within an array store. - -The metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. -shape - A list of integers defining the length of each dimension of the array. -chunks - A list of integers defining the length of each dimension of a chunk of the - array. Note that all chunks within a Zarr array have the same shape. -dtype - A string or list defining a valid data type for the array. See also - the subsection below on data type encoding. -compressor - A JSON object identifying the primary compression codec and providing - configuration parameters, or ``null`` if no compressor is to be used. - The object MUST contain an ``"id"`` key identifying the codec to be used. -fill_value - A scalar value providing the default value to use for uninitialized - portions of the array, or ``null`` if no fill_value is to be used. -order - Either "C" or "F", defining the layout of bytes within each chunk of the - array. "C" means row-major order, i.e., the last dimension varies fastest; - "F" means column-major order, i.e., the first dimension varies fastest. -filters - A list of JSON objects providing codec configurations, or ``null`` if no - filters are to be applied. Each codec configuration object MUST contain a - ``"id"`` key identifying the codec to be used. - -The following keys MAY be present within the object: - -dimension_separator - If present, either the string ``"."`` or ``"/"`` defining the separator placed - between the dimensions of a chunk. If the value is not set, then the - default MUST be assumed to be ``"."``, leading to chunk keys of the form "0.0". - Arrays defined with ``"/"`` as the dimension separator can be considered to have - nested, or hierarchical, keys of the form "0/0" that SHOULD where possible - produce a directory-like structure. - -Other keys SHOULD NOT be present within the metadata object and SHOULD be -ignored by implementations. - -For example, the JSON object below defines a 2-dimensional array of 64-bit -little-endian floating point numbers with 10000 rows and 10000 columns, divided -into chunks of 1000 rows and 1000 columns (so there will be 100 chunks in total -arranged in a 10 by 10 grid). Within each chunk the data are laid out in C -contiguous order. Each chunk is encoded using a delta filter and compressed -using the Blosc compression library prior to storage:: - - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "id": "blosc", - "cname": "lz4", - "clevel": 5, - "shuffle": 1 - }, - "dtype": "`. The format -consists of 3 parts: - -* One character describing the byteorder of the data (``"<"``: little-endian; - ``">"``: big-endian; ``"|"``: not-relevant) -* One character code giving the basic type of the array (``"b"``: Boolean (integer - type where all values are only True or False); ``"i"``: integer; ``"u"``: unsigned - integer; ``"f"``: floating point; ``"c"``: complex floating point; ``"m"``: timedelta; - ``"M"``: datetime; ``"S"``: string (fixed-length sequence of char); ``"U"``: unicode - (fixed-length sequence of Py_UNICODE); ``"V"``: other (void * – each item is a - fixed-size chunk of memory)) -* An integer specifying the number of bytes the type uses. - -The byte order MUST be specified. E.g., ``"i4"``, ``"|b1"`` and -``"|S12"`` are valid data type encodings. - -For datetime64 ("M") and timedelta64 ("m") data types, these MUST also include the -units within square brackets. A list of valid units and their definitions are given in -the :ref:`NumPy documentation on Datetimes and Timedeltas -`. -For example, ``"`. Each -sub-list has the form ``[fieldname, datatype, shape]`` where ``shape`` -is optional. ``fieldname`` is a string, ``datatype`` is a string -specifying a simple data type (see above), and ``shape`` is a list of -integers specifying subarray shape. For example, the JSON list below -defines a data type composed of three single-byte unsigned integer -fields named "r", "g" and "b":: - - [["r", "|u1"], ["g", "|u1"], ["b", "|u1"]] - -For example, the JSON list below defines a data type composed of three -fields named "x", "y" and "z", where "x" and "y" each contain 32-bit -floats, and each item in "z" is a 2 by 2 array of floats:: - - [["x", "`_ -produces a sequence of bytes that begins with a 16-byte header followed by -compressed data. - -The compressed sequence of bytes for each chunk is stored under a key formed -from the index of the chunk within the grid of chunks representing the array. -To form a string key for a chunk, the indices are converted to strings and -concatenated with the period character (".") separating each index. For -example, given an array with shape (10000, 10000) and chunk shape (1000, 1000) -there will be 100 chunks laid out in a 10 by 10 grid. The chunk with indices -(0, 0) provides data for rows 0-999 and columns 0-999 and is stored under the -key "0.0"; the chunk with indices (2, 4) provides data for rows 2000-2999 and -columns 4000-4999 and is stored under the key "2.4"; etc. - -There is no need for all chunks to be present within an array store. If a chunk -is not present then it is considered to be in an uninitialized state. An -uninitialized chunk MUST be treated as if it was uniformly filled with the value -of the "fill_value" field in the array metadata. If the "fill_value" field is -``null`` then the contents of the chunk are undefined. - -Note that all chunks in an array have the same shape. If the length of any -array dimension is not exactly divisible by the length of the corresponding -chunk dimension then some chunks will overhang the edge of the array. The -contents of any chunk region falling outside the array are undefined. - -.. _spec_v2_array_filters: - -Filters -~~~~~~~ - -Optionally a sequence of one or more filters can be used to transform chunk -data prior to compression. When storing data, filters are applied in the order -specified in array metadata to encode data, then the encoded data are passed to -the primary compressor. When retrieving data, stored chunk data are -decompressed by the primary compressor then decoded using filters in the -reverse order. - -.. _spec_v2_hierarchy: - -Hierarchies ------------ - -.. _spec_v2_hierarchy_paths: - -Logical storage paths -~~~~~~~~~~~~~~~~~~~~~ - -Multiple arrays can be stored in the same array store by associating each array -with a different logical path. A logical path is simply an ASCII string. The -logical path is used to form a prefix for keys used by the array. For example, -if an array is stored at logical path "foo/bar" then the array metadata will be -stored under the key "foo/bar/.zarray", the user-defined attributes will be -stored under the key "foo/bar/.zattrs", and the chunks will be stored under -keys like "foo/bar/0.0", "foo/bar/0.1", etc. - -To ensure consistent behaviour across different storage systems, logical paths -MUST be normalized as follows: - -* Replace all backward slash characters ("\\\\") with forward slash characters - ("/") -* Strip any leading "/" characters -* Strip any trailing "/" characters -* Collapse any sequence of more than one "/" character into a single "/" - character - -The key prefix is then obtained by appending a single "/" character to the -normalized logical path. - -After normalization, if splitting a logical path by the "/" character results -in any path segment equal to the string "." or the string ".." then an error -MUST be raised. - -N.B., how the underlying array store processes requests to store values under -keys containing the "/" character is entirely up to the store implementation -and is not constrained by this specification. E.g., an array store could simply -treat all keys as opaque ASCII strings; equally, an array store could map -logical paths onto some kind of hierarchical storage (e.g., directories on a -file system). - -.. _spec_v2_hierarchy_groups: - -Groups -~~~~~~ - -Arrays can be organized into groups which can also contain other groups. A -group is created by storing group metadata under the ".zgroup" key under some -logical path. E.g., a group exists at the root of an array store if the -".zgroup" key exists in the store, and a group exists at logical path "foo/bar" -if the "foo/bar/.zgroup" key exists in the store. - -If the user requests a group to be created under some logical path, then groups -MUST also be created at all ancestor paths. E.g., if the user requests group -creation at path "foo/bar" then groups MUST be created at path "foo" and the -root of the store, if they don't already exist. - -If the user requests an array to be created under some logical path, then -groups MUST also be created at all ancestor paths. E.g., if the user requests -array creation at path "foo/bar/baz" then groups must be created at path -"foo/bar", path "foo", and the root of the store, if they don't already exist. - -The group metadata resource is a JSON object. The following keys MUST be present -within the object: - -zarr_format - An integer defining the version of the storage specification to which the - array store adheres. - -Other keys MUST NOT be present within the metadata object. - -The members of a group are arrays and groups stored under logical paths that -are direct children of the parent group's logical path. E.g., if groups exist -under the logical paths "foo" and "foo/bar" and an array exists at logical path -"foo/baz" then the members of the group at path "foo" are the group at path -"foo/bar" and the array at path "foo/baz". - -.. _spec_v2_attrs: - -Attributes ----------- - -An array or group can be associated with custom attributes, which are arbitrary -key/value pairs with application-specific meaning. Custom attributes are encoded -as a JSON object and stored under the ".zattrs" key within an array store. The -".zattrs" key does not have to be present, and if it is absent the attributes -should be treated as empty. - -For example, the JSON object below encodes three attributes named -"foo", "bar" and "baz":: - - { - "foo": 42, - "bar": "apples", - "baz": [1, 2, 3, 4] - } - -.. _spec_v2_examples: - -Examples --------- - -Storing a single array -~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing a Zarr array, using a directory on the -local file system as storage. - -Create an array:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/example.zarr') - >>> a = zarr.create(shape=(20, 20), chunks=(10, 10), dtype='i4', - ... fill_value=42, compressor=zarr.Zlib(level=1), - ... store=store, overwrite=True) - -No chunks are initialized yet, so only the ".zarray" and ".zattrs" keys -have been set in the store:: - - >>> import os - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray'] - -Inspect the array metadata:: - - >>> print(open('data/example.zarr/.zarray').read()) - { - "chunks": [ - 10, - 10 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": ">> a[0:10, 0:10] = 1 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0'] - -Set some more data:: - - >>> a[0:10, 10:20] = 2 - >>> a[10:20, :] = 3 - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - -Manually decompress a single chunk for illustration:: - - >>> import zlib - >>> buf = zlib.decompress(open('data/example.zarr/0.0', 'rb').read()) - >>> import numpy as np - >>> chunk = np.frombuffer(buf, dtype='>> chunk - array([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1], dtype=int32) - -Modify the array attributes:: - - >>> a.attrs['foo'] = 42 - >>> a.attrs['bar'] = 'apples' - >>> a.attrs['baz'] = [1, 2, 3, 4] - >>> sorted(os.listdir('data/example.zarr')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - >>> print(open('data/example.zarr/.zattrs').read()) - { - "bar": "apples", - "baz": [ - 1, - 2, - 3, - 4 - ], - "foo": 42 - } - -Storing multiple arrays in a hierarchy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Below is an example of storing multiple Zarr arrays organized into a group -hierarchy, using a directory on the local file system as storage. This storage -implementation maps logical paths onto directory paths on the file system, -however this is an implementation choice and is not required. - -Setup the store:: - - >>> import zarr - >>> store = zarr.DirectoryStore('data/group.zarr') - -Create the root group:: - - >>> root_grp = zarr.group(store, overwrite=True) - -The metadata resource for the root group has been created:: - - >>> import os - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup'] - -Inspect the group metadata:: - - >>> print(open('data/group.zarr/.zgroup').read()) - { - "zarr_format": 2 - } - -Create a sub-group:: - - >>> sub_grp = root_grp.create_group('foo') - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup'] - -Create an array within the sub-group:: - - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - -Set a custom attributes:: - - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - -What has been stored:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '.zattrs', '0.0', '0.1', '1.0', '1.1'] - -Here is the same example using a Zip file as storage:: - - >>> store = zarr.ZipStore('data/group.zip', mode='w') - >>> root_grp = zarr.group(store) - >>> sub_grp = root_grp.create_group('foo') - >>> a = sub_grp.create_dataset('bar', shape=(20, 20), chunks=(10, 10)) - >>> a[:] = 42 - >>> a.attrs['comment'] = 'answer to life, the universe and everything' - >>> store.close() - -What has been stored:: - - >>> import zipfile - >>> zf = zipfile.ZipFile('data/group.zip', mode='r') - >>> for name in sorted(zf.namelist()): - ... print(name) - .zgroup - foo/.zgroup - foo/bar/.zarray - foo/bar/.zattrs - foo/bar/0.0 - foo/bar/0.1 - foo/bar/1.0 - foo/bar/1.1 - -.. _spec_v2_changes: - -Changes -------- - -Version 2 clarifications -~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes have been made to the version 2 specification since it was -initially published to clarify ambiguities and add some missing information. - -* The specification now describes how bytes fill values should be encoded and - decoded for arrays with a fixed-length byte string data type (:issue:`165`, - :issue:`176`). - -* The specification now clarifies that units must be specified for datetime64 and - timedelta64 data types (:issue:`85`, :issue:`215`). - -* The specification now clarifies that the '.zattrs' key does not have to be present for - either arrays or groups, and if absent then custom attributes should be treated as - empty. - -* The specification now describes how structured datatypes with - subarray shapes and/or with nested structured data types are encoded - in array metadata (:issue:`111`, :issue:`296`). - -* Clarified the key/value pairs of custom attributes as "arbitrary" rather than - "simple". - -Changes from version 1 to version 2 -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following changes were made between version 1 and version 2 of this specification: - -* Added support for storing multiple arrays in the same store and organising - arrays into hierarchies using groups. -* Array metadata is now stored under the ".zarray" key instead of the "meta" - key. -* Custom attributes are now stored under the ".zattrs" key instead of the - "attrs" key. -* Added support for filters. -* Changed encoding of "fill_value" field within array metadata. -* Changed encoding of compressor information within array metadata to be - consistent with representation of filter information. +The V2 Specification has been migrated to its website → +https://zarr-specs.readthedocs.io/. diff --git a/docs/spec/v3.rst b/docs/spec/v3.rst index bd8852707b..3d39f35ba6 100644 --- a/docs/spec/v3.rst +++ b/docs/spec/v3.rst @@ -1,7 +1,7 @@ .. _spec_v3: Zarr Storage Specification Version 3 -======================================================= +==================================== The V3 Specification has been migrated to its website → https://zarr-specs.readthedocs.io/. From 97cb3a7361ab763d6c2a7ab1128780f9142bd6aa Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 23:46:31 +0200 Subject: [PATCH 0066/1043] chore: update pre-commit hooks (#1738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.4 → v0.3.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.4...v0.3.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6c2762f34d..24ff72a12f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.4' + rev: 'v0.3.5' hooks: - id: ruff - repo: https://github.com/psf/black From 6ef6714bc63d66c409cf5780791d6349081fcd45 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 1 Apr 2024 23:13:42 -0600 Subject: [PATCH 0067/1043] Optimize Array.info and Group.info (#1733) * Optimize Array.info. Avoid repeated computes of the same value (getsize) * Don't have InfoReporter query items twice. Apparently IPython will run both __repr__ and _repr_html_ so we were calling `getsize` twice. * Group too * Apply suggestions from code review Co-authored-by: Joe Hamman --------- Co-authored-by: Joe Hamman --- docs/release.rst | 2 ++ zarr/core.py | 13 +++++++------ zarr/hierarchy.py | 3 +-- zarr/util.py | 7 +++---- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 9c75dc4feb..736838cdaf 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -21,6 +21,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* Optimize ``Array.info`` so that it calls `getsize` only once. + By :user:`Deepak Cherian `. * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. diff --git a/zarr/core.py b/zarr/core.py index c3184c6652..07f38cd96d 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -176,7 +176,6 @@ def __init__( ) # initialize info reporter - self._info_reporter = InfoReporter(self) # initialize indexing helpers self._oindex = OIndex(self) @@ -2429,7 +2428,7 @@ def info(self): Chunks initialized : 0/10 """ - return self._info_reporter + return InfoReporter(self) def info_items(self): return self._synchronized_op(self._info_items_nosync) @@ -2471,14 +2470,16 @@ def bytestr(n): items += [("Synchronizer type", typestr(self._synchronizer))] # storage info + nbytes = self.nbytes + nbytes_stored = self.nbytes_stored items += [("Store type", typestr(self._store))] if self._chunk_store is not None: items += [("Chunk store type", typestr(self._chunk_store))] - items += [("No. bytes", bytestr(self.nbytes))] - if self.nbytes_stored > 0: + items += [("No. bytes", bytestr(nbytes))] + if nbytes_stored > 0: items += [ - ("No. bytes stored", bytestr(self.nbytes_stored)), - ("Storage ratio", f"{self.nbytes / self.nbytes_stored:.1f}"), + ("No. bytes stored", bytestr(nbytes_stored)), + ("Storage ratio", f"{nbytes / nbytes_stored:.1f}"), ] items += [("Chunks initialized", f"{self.nchunks_initialized}/{self.nchunks}")] diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index c88892c932..c5f7a37bc6 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -211,7 +211,6 @@ def __init__( ) # setup info - self._info = InfoReporter(self) @property def store(self): @@ -266,7 +265,7 @@ def attrs(self): @property def info(self): """Return diagnostic information about the group.""" - return self._info + return InfoReporter(self) @property def meta_array(self): diff --git a/zarr/util.py b/zarr/util.py index dc8aff0edf..848f1ed114 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -408,14 +408,13 @@ def info_html_report(items) -> str: class InfoReporter: def __init__(self, obj): self.obj = obj + self.items = self.obj.info_items() def __repr__(self): - items = self.obj.info_items() - return info_text_report(items) + return info_text_report(self.items) def _repr_html_(self): - items = self.obj.info_items() - return info_html_report(items) + return info_html_report(self.items) class TreeNode: From 9864b40731a11ed826f6b69e6b9e381a12168096 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Apr 2024 05:29:11 +0000 Subject: [PATCH 0068/1043] Bump actions/setup-python from 5.0.0 to 5.1.0 (#1736) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index fe168d2862..8ac76c899b 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.0.0 + - uses: actions/setup-python@v5.1.0 name: Install Python with: python-version: '3.9' From aa9a0d5e1874217bcaa15340860cb8b606944fb6 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 2 Apr 2024 19:45:48 +0200 Subject: [PATCH 0069/1043] Couple fixes (#1737) * Use `is` when comparing `type` of two objects * Unnecessary `None` provided as default --- zarr/_storage/store.py | 2 +- zarr/tests/test_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 209f118534..0a08080548 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -462,7 +462,7 @@ def inner_store(self) -> Union["StorageTransformer", StoreV3]: def __eq__(self, other): return ( - type(self) == type(other) + type(self) is type(other) and self._inner_store == other._inner_store and self.get_config() == other.get_config() ) diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index d9447c0832..730f724314 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -123,7 +123,7 @@ def create_array(self, shape: Union[int, Tuple[int, ...]], **kwargs): "compressor": kwargs.pop("compressor", self.compressor), "chunk_store": chunk_store, "storage_transformers": self.create_storage_transformers(shape), - "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype", None))), + "filters": kwargs.pop("filters", self.create_filters(kwargs.get("dtype"))), } # keyword arguments for array instantiation From d0fb8758483dd811234300eea7a1f24909bb929f Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 3 Apr 2024 20:18:18 +0200 Subject: [PATCH 0070/1043] Fix tests with Pytest 8 (#1714) * Bump pytest version * Use editable install when testing --------- Co-authored-by: Sanket Verma --- .github/workflows/python-package.yml | 2 +- requirements_dev_minimal.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 2f9166ae96..a37fa3c63a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -64,7 +64,7 @@ jobs: python -m pip install --upgrade pip python -m pip install -U pip setuptools wheel line_profiler python -m pip install -rrequirements_dev_minimal.txt numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis - python -m pip install . + python -m pip install -e . python -m pip freeze - name: Tests shell: "bash -l {0}" diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 94d3fff8a6..5d156db655 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,4 @@ numcodecs==0.12.1 msgpack-python==0.5.6 setuptools-scm==8.0.4 # test requirements -pytest==7.4.4 +pytest==8.1.1 From bcb7684b5a15d4ead21e305301581e02193e8677 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 16:17:08 -0600 Subject: [PATCH 0071/1043] Avoid redundant __contains__ (#1739) Let's try grabbing the array.json and group.json files, and check for `*NotFoundError`, instead of using contains first. Co-authored-by: Davis Bennett --- zarr/hierarchy.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index c5f7a37bc6..0067eaebb5 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -27,6 +27,7 @@ from zarr.errors import ( ContainsArrayError, ContainsGroupError, + ArrayNotFoundError, GroupNotFoundError, ReadOnlyError, ) @@ -457,7 +458,7 @@ def __getitem__(self, item): """ path = self._item_path(item) - if contains_array(self._store, path): + try: return Array( self._store, read_only=self._read_only, @@ -468,7 +469,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif contains_group(self._store, path, explicit_only=True): + except ArrayNotFoundError: + pass + + try: return Group( self._store, read_only=self._read_only, @@ -479,7 +483,10 @@ def __getitem__(self, item): zarr_version=self._version, meta_array=self._meta_array, ) - elif self._version == 3: + except GroupNotFoundError: + pass + + if self._version == 3: implicit_group = meta_root + path + "/" # non-empty folder in the metadata path implies an implicit group if self._store.list_prefix(implicit_group): From 0cfd2be24feb7cd86002b9ab22cd65f65ed85e3a Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 20:13:23 -0600 Subject: [PATCH 0072/1043] Array & Group: Use already loaded attributes to populate cache. (#1734) * Array: Use already loaded attributes to populate cache. * Group: Use already loaded attributes to populate cache. * Fix * Add release note --- docs/release.rst | 4 ++++ zarr/attrs.py | 6 ++++-- zarr/core.py | 7 ++++++- zarr/hierarchy.py | 7 ++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 736838cdaf..346d673d68 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -21,8 +21,12 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* [v3] Reuse the download array metadata when creating an ``Array``. + By :user:`Deepak Cherian `. + * Optimize ``Array.info`` so that it calls `getsize` only once. By :user:`Deepak Cherian `. + * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. diff --git a/zarr/attrs.py b/zarr/attrs.py index e967c5b853..2afcaf295e 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -25,14 +25,16 @@ class Attributes(MutableMapping): """ - def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None): + def __init__( + self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None, cached_dict=None + ): self._version = getattr(store, "_store_version", 2) _Store = Store if self._version == 2 else StoreV3 self.store = _Store._ensure_store(store) self.key = key self.read_only = read_only self.cache = cache - self._cached_asdict = None + self._cached_asdict = cached_dict if cache else None self.synchronizer = synchronizer def _get_nosync(self): diff --git a/zarr/core.py b/zarr/core.py index 07f38cd96d..1bd081acee 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -172,7 +172,12 @@ def __init__( # initialize attributes akey = _prefix_to_attrs_key(self._store, self._key_prefix) self._attrs = Attributes( - store, key=akey, read_only=read_only, synchronizer=synchronizer, cache=cache_attrs + store, + key=akey, + read_only=read_only, + synchronizer=synchronizer, + cache=cache_attrs, + cached_dict=self._meta["attributes"] if self._version == 3 else None, ) # initialize info reporter diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 0067eaebb5..0fb07dd620 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -208,7 +208,12 @@ def __init__( # object can still be created. akey = mkey self._attrs = Attributes( - store, key=akey, read_only=read_only, cache=cache_attrs, synchronizer=synchronizer + store, + key=akey, + read_only=read_only, + cache=cache_attrs, + synchronizer=synchronizer, + cached_dict=self._meta["attributes"] if self._version == 3 and self._meta else None, ) # setup info From 62910bcebd3a583cea79af2077fa1d1798845797 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 4 Apr 2024 20:14:09 -0600 Subject: [PATCH 0073/1043] Optimize attribute setting (#1741) * Optimize attribute setting * Add release note --------- Co-authored-by: Davis Bennett --- docs/release.rst | 3 +++ zarr/attrs.py | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 346d673d68..b011b0986b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -20,6 +20,9 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* [v3] Dramatically reduce number of ``__contains_`` requests in favor of optimistically calling `__getitem__` + and handling any error that may arise. + By :user:`Deepak Cherian `. * [v3] Reuse the download array metadata when creating an ``Array``. By :user:`Deepak Cherian `. diff --git a/zarr/attrs.py b/zarr/attrs.py index 2afcaf295e..af9a5f1d30 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -151,19 +151,20 @@ def _put_nosync(self, d): if self.cache: self._cached_asdict = d else: - if self.key in self.store: + try: + meta_unparsed = self.store[self.key] # Cannot write the attributes directly to JSON, but have to # store it within the pre-existing attributes key of the v3 # metadata. # Note: this changes the store.counter result in test_caching_on! - meta = self.store._metadata_class.parse_metadata(self.store[self.key]) + meta = self.store._metadata_class.parse_metadata(meta_unparsed) if "attributes" in meta and "filters" in meta["attributes"]: # need to preserve any existing "filters" attribute d["attributes"]["filters"] = meta["attributes"]["filters"] meta["attributes"] = d["attributes"] - else: + except KeyError: meta = d self.store[self.key] = json_dumps(meta) if self.cache: From 5fde3a29ca0a9f1a005eb5000d1e8585d4ca0bde Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Fri, 5 Apr 2024 13:15:10 -0400 Subject: [PATCH 0074/1043] Make sure fs exceptions are raised if not MissingFs exceptions (clone) (#1604) * Make sure fs exceptions are raised if not Missing * lint * add missing argument in tests, lint * clear memory filesystem during test * improve commenting * add memory_store fixture, getitems performance * Update release.rst * improve FSStore.test_exception coverage --------- Co-authored-by: Martin Durant Co-authored-by: Joe Hamman Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --- docs/release.rst | 3 +++ zarr/storage.py | 22 +++++++++++++++++----- zarr/tests/test_storage.py | 25 +++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index b011b0986b..da802651c2 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -41,6 +41,9 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. + +* FSStore now raises rather than return bad data. + By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. .. _release_2.17.1: diff --git a/zarr/storage.py b/zarr/storage.py index f6903d29b2..10f55f0ba3 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -1417,11 +1417,23 @@ def _normalize_key(self, key): def getitems( self, keys: Sequence[str], *, contexts: Mapping[str, Context] ) -> Mapping[str, Any]: - keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") - # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back into the original key space. - return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} + keys_transformed = {self._normalize_key(key): key for key in keys} + results_transformed = self.map.getitems(list(keys_transformed), on_error="return") + results = {} + for k, v in results_transformed.items(): + if isinstance(v, self.exceptions): + # Cause recognized exceptions to prompt a KeyError in the + # function calling this method + continue + elif isinstance(v, Exception): + # Raise any other exception + raise v + else: + # The function calling this method may not recognize the transformed + # keys, so we send the values returned by self.map.getitems back into + # the original key space. + results[keys_transformed[k]] = v + return results def __getitem__(self, key): key = self._normalize_key(key) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index 358d043ad6..ae8a56fa61 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1098,6 +1098,12 @@ def mock_walker_no_slash(_path): @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStore(StoreTests): + @pytest.fixture + def memory_store(self): + store = FSStore("memory://") + yield store + store.fs.store.clear() + def create_store(self, normalize_keys=False, dimension_separator=".", path=None, **kwargs): if path is None: path = tempfile.mkdtemp() @@ -1337,6 +1343,25 @@ def test_s3_complex(self): ) assert (a[:] == -np.ones((8, 8, 8))).all() + def test_exceptions(self, memory_store): + fs = memory_store.fs + group = zarr.open(memory_store, mode="w") + x = group.create_dataset("x", data=[1, 2, 3]) + y = group.create_dataset("y", data=1) + fs.store["/x/0"] = None + fs.store["/y/0"] = None + # no exception from FSStore.getitems getting KeyError + assert group.store.getitems(["foo"], contexts={}) == {} + # exception from FSStore.getitems getting AttributeError + with pytest.raises(Exception): + group.store.getitems(["x/0"], contexts={}) + # exception from FSStore.getitems getting AttributeError + with pytest.raises(Exception): + x[...] + # exception from FSStore.__getitem__ getting AttributeError + with pytest.raises(Exception): + y[...] + @pytest.mark.skipif(have_fsspec is False, reason="needs fsspec") class TestFSStoreWithKeySeparator(StoreTests): From 1631c109c655c71221eab6d964063abd015011f2 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 5 Apr 2024 13:53:33 -0700 Subject: [PATCH 0075/1043] chore(release): update changelog for 2.17.2 (#1775) --- docs/release.rst | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index da802651c2..60d01b8244 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,18 +13,18 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. -.. _unreleased: +.. _release_2.17.2: -Unreleased ----------- +2.17.2 +------ Enhancements ~~~~~~~~~~~~ -* [v3] Dramatically reduce number of ``__contains_`` requests in favor of optimistically calling `__getitem__` +* [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` and handling any error that may arise. By :user:`Deepak Cherian `. -* [v3] Reuse the download array metadata when creating an ``Array``. +* [v3] Reuse the downloaded array metadata when creating an ``Array``. By :user:`Deepak Cherian `. * Optimize ``Array.info`` so that it calls `getsize` only once. @@ -33,6 +33,9 @@ Enhancements * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. +* FSStore now raises rather than return bad data. + By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. + Maintenance ~~~~~~~~~~~ @@ -42,8 +45,6 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. -* FSStore now raises rather than return bad data. - By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. .. _release_2.17.1: From 6105ef203e3e5c5390aa01db613bc42e8fb98c1a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 6 Apr 2024 11:50:52 -0700 Subject: [PATCH 0076/1043] chore(docs): reset release notes as unreleased (#1776) --- docs/release.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 60d01b8244..75193bc3e3 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,6 +13,23 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. +.. _unreleased: + +Unreleased +---------- + +Enhancements +~~~~~~~~~~~~ + + +Docs +~~~~ + + +Maintenance +~~~~~~~~~~~ + + .. _release_2.17.2: 2.17.2 From 5d566eb16db0f63989816b1300f1ec50927f6632 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 18:58:44 +0200 Subject: [PATCH 0077/1043] Bump codecov/codecov-action from 3 to 4 (#1647) * Bump codecov/codecov-action from 3 to 4 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] * Set codecov env --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Josh Moore --- .github/workflows/python-package.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index a37fa3c63a..8ff6e9a2eb 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -80,11 +80,8 @@ jobs: mkdir ~/blob_emulator azurite -l ~/blob_emulator --debug debug.log 2>&1 > stdouterr.log & pytest --cov=zarr --cov-config=pyproject.toml --doctest-plus --cov-report xml --cov=./ --timeout=300 - - uses: codecov/codecov-action@v3 + - uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos - #files: ./coverage1.xml,./coverage2.xml # optional - #flags: unittests # optional - #name: codecov-umbrella # optional - #fail_ci_if_error: true # optional (default = false) verbose: true # optional (default = false) From b98f6941912da99dcd8350c49a5306e74af05e14 Mon Sep 17 00:00:00 2001 From: Sanket Verma Date: Mon, 22 Apr 2024 21:26:31 +0200 Subject: [PATCH 0078/1043] Update release.rst for v2.17.2 (#1778) * Update release.rst for v2.17.2 * Minor edits --------- Co-authored-by: Joe Hamman --- docs/release.rst | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 75193bc3e3..3b5ab631df 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -37,15 +37,16 @@ Maintenance Enhancements ~~~~~~~~~~~~ + * [v3] Dramatically reduce number of ``__contains__`` requests in favor of optimistically calling `__getitem__` and handling any error that may arise. - By :user:`Deepak Cherian `. + By :user:`Deepak Cherian ` :issue:`1741`. * [v3] Reuse the downloaded array metadata when creating an ``Array``. - By :user:`Deepak Cherian `. + By :user:`Deepak Cherian ` :issue:`1734`. * Optimize ``Array.info`` so that it calls `getsize` only once. - By :user:`Deepak Cherian `. + By :user:`Deepak Cherian ` :issue:`1733`. * Override IPython ``_repr_*_`` methods to avoid expensive lookups against object stores. By :user:`Deepak Cherian ` :issue:`1716`. @@ -53,6 +54,21 @@ Enhancements * FSStore now raises rather than return bad data. By :user:`Martin Durant ` and :user:`Ian Carroll ` :issue:`1604`. +* Avoid redundant ``__contains__``. + By :user:`Deepak Cherian ` :issue:`1739`. + +Docs +~~~~ + +* Fix link to GCSMap in ``tutorial.rst``. + By :user:`Daniel Jahn ` :issue:`1689`. + +* Endorse `SPEC0000 `_ and state version support policy in ``installation.rst``. + By :user:`Sanket Verma ` :issue:`1665`. + +* Migrate v1 and v2 specification to `Zarr-Specs `_. + By :user:`Sanket Verma ` :issue:`1582`. + Maintenance ~~~~~~~~~~~ @@ -61,7 +77,12 @@ Maintenance * Bump minimum supported NumPy version to 1.23 (per spec 0000) By :user:`Joe Hamman ` :issue:`1719`. - + +* Minor fixes: Using ``is`` instead of ``type`` and removing unnecessary ``None``. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1737`. + +* Fix tests failure related to Pytest 8. + By :user:`David Stansby ` :issue:`1714`. .. _release_2.17.1: From dd3dd96c7a5b7891bfa6be6ff9e989b49d00b0f5 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 22 Apr 2024 12:45:07 -0700 Subject: [PATCH 0079/1043] Deprecate the experimental v3 implementation (#1802) * deprecate(exp-v3): Add a future warning about the pending removal of the experimental v3 implementation * ignore warning * add test --- pyproject.toml | 1 + zarr/_storage/store.py | 13 +++++++++++++ zarr/tests/test_storage_v3.py | 20 +++++++++++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0be79f990e..904c974424 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ filterwarnings = [ "error:::zarr.*", "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", + "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", ] diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 0a08080548..69986ecadd 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -1,5 +1,6 @@ import abc import os +import warnings from collections import defaultdict from collections.abc import MutableMapping from copy import copy @@ -23,9 +24,21 @@ DEFAULT_ZARR_VERSION: ZARR_VERSION = 2 v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"] +_has_warned_about_v3 = False # to avoid printing the warning multiple times def assert_zarr_v3_api_available(): + # we issue a warning about the experimental v3 implementation when it is first used + global _has_warned_about_v3 + if v3_api_available and not _has_warned_about_v3: + warnings.warn( + "The experimental Zarr V3 implementation in this version of Zarr-Python is not " + "in alignment with the final V3 specification. This version will be removed in " + "Zarr-Python 3 in favor of a spec compliant version.", + FutureWarning, + stacklevel=1, + ) + _has_warned_about_v3 = True if not v3_api_available: raise NotImplementedError( "# V3 reading and writing is experimental! To enable support, set:\n" diff --git a/zarr/tests/test_storage_v3.py b/zarr/tests/test_storage_v3.py index e15b2db743..c096f9cb02 100644 --- a/zarr/tests/test_storage_v3.py +++ b/zarr/tests/test_storage_v3.py @@ -4,12 +4,18 @@ import inspect import os import tempfile +import warnings import numpy as np import pytest import zarr -from zarr._storage.store import _get_hierarchy_metadata, v3_api_available, StorageTransformer +from zarr._storage.store import ( + _get_hierarchy_metadata, + assert_zarr_v3_api_available, + v3_api_available, + StorageTransformer, +) from zarr._storage.v3_storage_transformers import ShardingStorageTransformer, v3_sharding_available from zarr.core import Array from zarr.meta import _default_entry_point_metadata_v3 @@ -668,6 +674,18 @@ def test_top_level_imports(): assert not hasattr(zarr, store_name) # pragma: no cover +def test_assert_zarr_v3_api_available_warns_once(): + import zarr._storage.store + + zarr._storage.store._has_warned_about_v3 = False + warnings.resetwarnings() + with pytest.warns() as record: + assert_zarr_v3_api_available() + assert_zarr_v3_api_available() + assert len(record) == 1 + assert "The experimental Zarr V3 implementation" in str(record[0].message) + + def _get_public_and_dunder_methods(some_class): return set( name From 0a29fb3e40e57dd8de62f786a18c807982d5a2da Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 20:26:11 +0000 Subject: [PATCH 0080/1043] chore: update pre-commit hooks (#1779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.3.5 → v0.3.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.3.5...v0.3.7) - [github.com/psf/black: 24.3.0 → 24.4.0](https://github.com/psf/black/compare/24.3.0...24.4.0) - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 24ff72a12f..0aa13b31a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.3.5' + rev: 'v0.4.1' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.3.0 + rev: 24.4.0 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -20,7 +20,7 @@ repos: hooks: - id: codespell - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy From 9d046ea0d2878af7d15b3de3ec3036fe31661340 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 22 Apr 2024 14:34:17 -0600 Subject: [PATCH 0081/1043] Fix `is_total_slice` for size-1 dimensions (#1800) Closes #1730 Co-authored-by: Ryan Abernathey Co-authored-by: Joe Hamman --- docs/release.rst | 2 ++ zarr/tests/test_util.py | 9 +++++++++ zarr/util.py | 13 +++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 3b5ab631df..07c2a47e7c 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -20,6 +20,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ +* Performance improvement for reading and writing chunks if any of the dimensions is size 1. :issue:`1730` + By :user:`Deepak Cherian `. Docs diff --git a/zarr/tests/test_util.py b/zarr/tests/test_util.py index 1f7efc9214..d908c7b2d7 100644 --- a/zarr/tests/test_util.py +++ b/zarr/tests/test_util.py @@ -89,6 +89,15 @@ def test_is_total_slice(): assert not is_total_slice((slice(0, 50), slice(0, 50)), (100, 100)) assert not is_total_slice((slice(0, 100, 2), slice(0, 100)), (100, 100)) + # size-1 dimension edge-case + # https://github.com/zarr-developers/zarr-python/issues/1730 + assert is_total_slice((slice(0, 1),), (1,)) + # this is an equivalent selection (without a slice) + assert is_total_slice((0,), (1,)) + # same for multidimensional selection + assert is_total_slice((slice(0, 1), slice(0, 10)), (1, 10)) + assert is_total_slice((0, slice(0, 10)), (1, 10)) + with pytest.raises(TypeError): is_total_slice("foo", (100,)) diff --git a/zarr/util.py b/zarr/util.py index 848f1ed114..e58aed80ab 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -234,8 +234,17 @@ def is_total_slice(item, shape: Tuple[int]) -> bool: if isinstance(item, tuple): return all( ( - isinstance(it, slice) - and ((it == slice(None)) or ((it.stop - it.start == sh) and (it.step in [1, None]))) + ( + isinstance(it, slice) + and ( + (it == slice(None)) + or ((it.stop - it.start == sh) and (it.step in [1, None])) + ) + ) + # The only scalar edge case, indexing with int 0 along a size-1 dimension + # is identical to a total slice + # https://github.com/zarr-developers/zarr-python/issues/1730 + or (isinstance(it, int) and it == 0 and sh == 1) ) for it, sh in zip(item, shape) ) From 60b53b059e0d5b4fd57dab74520af0184da32006 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 25 Apr 2024 06:14:40 -0700 Subject: [PATCH 0082/1043] add note to the top of the release page noting the plan for 2.18.* and 3.0 (#1816) --- docs/release.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 07c2a47e7c..1c4e52f7c0 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -13,6 +13,11 @@ Release notes # to document your changes. On releases it will be # re-indented so that it does not show up in the notes. +.. note:: + Zarr-Python 2.18.* is expected be the final release in the 2.* series. Work on Zarr-Python 3.0 is underway. + See `GH1777 `_ for more details on the upcoming + 3.0 release. + .. _unreleased: Unreleased From f4f6e8692026201223c04850bbe13e0551471c39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Apr 2024 13:12:24 +0200 Subject: [PATCH 0083/1043] Bump conda-incubator/setup-miniconda from 3.0.3 to 3.0.4 (#1824) Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.3 to 3.0.4. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/v3.0.3...v3.0.4) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/minimal.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/windows-testing.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/minimal.yml b/.github/workflows/minimal.yml index dba6918514..b5b2f48d62 100644 --- a/.github/workflows/minimal.yml +++ b/.github/workflows/minimal.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.3 + uses: conda-incubator/setup-miniconda@v3.0.4 with: channels: conda-forge environment-file: environment.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8ff6e9a2eb..f53cb2d9a9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -44,7 +44,7 @@ jobs: with: fetch-depth: 0 - name: Setup Miniconda - uses: conda-incubator/setup-miniconda@v3.0.3 + uses: conda-incubator/setup-miniconda@v3.0.4 with: channels: conda-forge python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/windows-testing.yml b/.github/workflows/windows-testing.yml index d580ef3f0e..ab86831aae 100644 --- a/.github/workflows/windows-testing.yml +++ b/.github/workflows/windows-testing.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3.0.3 + - uses: conda-incubator/setup-miniconda@v3.0.4 with: auto-update-conda: true python-version: ${{ matrix.python-version }} From 29b4acc45ed7b1f865c31bfa5d7beda4b3b9269a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 30 Apr 2024 09:17:05 -0700 Subject: [PATCH 0084/1043] dep(docs): deprecate experimental v3 support in docs (#1807) * dep(docs): deprecate experimental v3 support in docs * Apply suggestions from code review Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --------- Co-authored-by: Josh Moore Co-authored-by: Sanket Verma --- docs/api/v3.rst | 9 ++++----- docs/release.rst | 6 ++++++ zarr/convenience.py | 25 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/api/v3.rst b/docs/api/v3.rst index 7665b2ddd1..3503e3fe81 100644 --- a/docs/api/v3.rst +++ b/docs/api/v3.rst @@ -1,13 +1,12 @@ V3 Specification Implementation(``zarr._storage.v3``) ===================================================== -This module contains the implementation of the `Zarr V3 Specification `_. +This module contains an experimental implementation of the `Zarr V3 Specification `_. .. warning:: - Since Zarr Python 2.12 release, this module provides experimental infrastructure for reading and - writing the upcoming V3 spec of the Zarr format. Users wishing to prepare for the migration can set - the environment variable ``ZARR_V3_EXPERIMENTAL_API=1`` to begin experimenting, however data - written with this API should be expected to become stale, as the implementation will still change. + The experimental v3 implementation included in Zarr Python >2.12,<3 is not aligned with the final + V3 specification. This version is deprecated and will be removed in Zarr Python 3.0 in favor of a + spec compliant version. The new ``zarr._store.v3`` package has the necessary classes and functions for evaluating Zarr V3. Since the design is not finalised, the classes and functions are not automatically imported into diff --git a/docs/release.rst b/docs/release.rst index 1c4e52f7c0..811ede3d58 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -36,6 +36,12 @@ Docs Maintenance ~~~~~~~~~~~ +Deprecations +~~~~~~~~~~~~ + +* Deprecate experimental v3 support by issuing a `FutureWarning`. + Also updated docs to warn about using the experimental v3 version. + By :user:`Joe Hamman ` :issue:`1802` and :issue: `1807`. .. _release_2.17.2: diff --git a/zarr/convenience.py b/zarr/convenience.py index 7ca5d426f0..bd284e0844 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -55,6 +55,11 @@ def open(store: StoreLike = None, mode: str = "a", *, zarr_version=None, path=No The zarr protocol version to use. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store to open. **kwargs @@ -150,6 +155,11 @@ def save_array(store: StoreLike, arr, *, zarr_version=None, path=None, **kwargs) The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store where the array will be saved. kwargs @@ -200,6 +210,11 @@ def save_group(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional Path within the store where the group will be saved. kwargs @@ -282,6 +297,11 @@ def save(store: StoreLike, *args, zarr_version=None, path=None, **kwargs): The zarr protocol version to use when saving. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the group where the arrays will be saved. kwargs @@ -395,6 +415,11 @@ def load(store: StoreLike, zarr_version=None, path=None): The zarr protocol version to use when loading. The default value of None will attempt to infer the version from `store` if possible, otherwise it will fall back to 2. + + .. warning:: `zarr_version=3` is currently using the experimental Zarr V3 + implementation. This implementation is not in sync with the final specification + and will be replaced with a spec compliant version in the version 3.0. + path : str or None, optional The path within the store from which to load. From 7fa17b8e1350e8fd254ea530c7806b484fe1fc60 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:26:02 +0000 Subject: [PATCH 0085/1043] Bump h5py from 3.10.0 to 3.11.0 (#1786) Bumps [h5py](https://github.com/h5py/h5py) from 3.10.0 to 3.11.0. - [Release notes](https://github.com/h5py/h5py/releases) - [Changelog](https://github.com/h5py/h5py/blob/master/docs/release_guide.rst) - [Commits](https://github.com/h5py/h5py/compare/3.10.0...3.11.0) --- updated-dependencies: - dependency-name: h5py dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Joe Hamman --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 809d1c0eee..951594460a 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -17,7 +17,7 @@ coverage pytest-cov==5.0.0 pytest-doctestplus==1.2.1 pytest-timeout==2.3.1 -h5py==3.10.0 +h5py==3.11.0 fsspec==2023.12.2 s3fs==2023.12.2 moto[server]>=5.0.1 From 9331430dfb1e348b8b2e5c6290cbce0956fd0cf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 May 2024 00:26:56 +0000 Subject: [PATCH 0086/1043] Bump redis from 5.0.3 to 5.0.4 (#1810) Bumps [redis](https://github.com/redis/redis-py) from 5.0.3 to 5.0.4. - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.3...v5.0.4) --- updated-dependencies: - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_optional.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 951594460a..3456cca21a 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -8,7 +8,7 @@ ipywidgets==8.1.2 # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.3 +redis==5.0.4 types-redis types-setuptools pymongo==4.6.3 From 2f2914456e391ec77032223e9587b1d3ef5608aa Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 6 May 2024 15:33:09 -0700 Subject: [PATCH 0087/1043] deprecate(stores): add deprecation warnings to stores that we plan to remove in v3 (#1801) * deprecate(stores): add deprecation warnings to DBMStore, LMDBStore, SQLiteStore, MongoDBStore, RedisStore, and ABSStore * filter warnings in pytest config * more deprecation warnings in docstrings * add release note --- docs/release.rst | 7 ++++ pyproject.toml | 1 + zarr/_storage/absstore.py | 15 ++++++++- zarr/_storage/store.py | 5 +++ zarr/storage.py | 68 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index 811ede3d58..5184ab5f9f 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -43,6 +43,13 @@ Deprecations Also updated docs to warn about using the experimental v3 version. By :user:`Joe Hamman ` :issue:`1802` and :issue: `1807`. +Deprecations +~~~~~~~~~~~~ +* Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, + :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, + and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. + By :user:`Joe Hamman ` :issue:`1801`. + .. _release_2.17.2: 2.17.2 diff --git a/pyproject.toml b/pyproject.toml index 904c974424..f2356480bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,6 +137,7 @@ filterwarnings = [ "error:::zarr.*", "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", + "ignore:The .* is deprecated and will be removed in a Zarr-Python version 3*:FutureWarning", "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", ] diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index 217b2a29e0..5d2606f2f2 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -5,7 +5,14 @@ from numcodecs.compat import ensure_bytes from zarr.util import normalize_storage_path -from zarr._storage.store import _get_metadata_suffix, data_root, meta_root, Store, StoreV3 +from zarr._storage.store import ( + _get_metadata_suffix, + data_root, + meta_root, + Store, + StoreV3, + V3_DEPRECATION_MESSAGE, +) from zarr.types import DIMENSION_SEPARATOR __doctest_requires__ = { @@ -73,6 +80,12 @@ def __init__( dimension_separator: Optional[DIMENSION_SEPARATOR] = None, client=None, ): + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=3, + ) + self._dimension_separator = dimension_separator self.prefix = normalize_storage_path(prefix) if client is None: diff --git a/zarr/_storage/store.py b/zarr/_storage/store.py index 69986ecadd..dba29d13c0 100644 --- a/zarr/_storage/store.py +++ b/zarr/_storage/store.py @@ -26,6 +26,11 @@ v3_api_available = os.environ.get("ZARR_V3_EXPERIMENTAL_API", "0").lower() not in ["0", "false"] _has_warned_about_v3 = False # to avoid printing the warning multiple times +V3_DEPRECATION_MESSAGE = ( + "The {store} is deprecated and will be removed in a Zarr-Python version 3, see " + "https://github.com/zarr-developers/zarr-python/issues/1274 for more information." +) + def assert_zarr_v3_api_available(): # we issue a warning about the experimental v3 implementation when it is first used diff --git a/zarr/storage.py b/zarr/storage.py index 10f55f0ba3..772fa7646a 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -88,6 +88,7 @@ DEFAULT_ZARR_VERSION, BaseStore, Store, + V3_DEPRECATION_MESSAGE, ) __doctest_requires__ = { @@ -1604,6 +1605,12 @@ class NestedDirectoryStore(DirectoryStore): special handling for chunk keys so that chunk files for multidimensional arrays are stored in a nested directory tree. + .. deprecated:: 2.18.0 + NestedDirectoryStore will be removed in Zarr-Python 3.0 where controlling + the chunk key encoding will be supported as part of the array metadata. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -1675,6 +1682,13 @@ class NestedDirectoryStore(DirectoryStore): def __init__( self, path, normalize_keys=False, dimension_separator: Optional[DIMENSION_SEPARATOR] = "/" ): + + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + super().__init__(path, normalize_keys=normalize_keys) if dimension_separator is None: dimension_separator = "/" @@ -1995,6 +2009,11 @@ def migrate_1to2(store): class DBMStore(Store): """Storage class using a DBM-style database. + .. deprecated:: 2.18.0 + DBMStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -2083,6 +2102,12 @@ def __init__( dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **open_kwargs, ): + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + if open is None: import dbm @@ -2200,6 +2225,10 @@ class LMDBStore(Store): """Storage class using LMDB. Requires the `lmdb `_ package to be installed. + .. deprecated:: 2.18.0 + LMDBStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. Parameters ---------- @@ -2261,6 +2290,12 @@ def __init__( ): import lmdb + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + # set default memory map size to something larger than the lmdb default, which is # very likely to be too small for any moderate array (logic copied from zict) map_size = 2**40 if sys.maxsize >= 2**32 else 2**28 @@ -2580,6 +2615,11 @@ def __delitem__(self, key): class SQLiteStore(Store): """Storage class using SQLite. + .. deprecated:: 2.18.0 + SQLiteStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Parameters ---------- path : string @@ -2612,6 +2652,12 @@ class SQLiteStore(Store): def __init__(self, path, dimension_separator: Optional[DIMENSION_SEPARATOR] = None, **kwargs): import sqlite3 + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._dimension_separator = dimension_separator # normalize path @@ -2778,6 +2824,11 @@ class MongoDBStore(Store): .. note:: This is an experimental feature. + .. deprecated:: 2.18.0 + MongoDBStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Requires the `pymongo `_ package to be installed. @@ -2810,6 +2861,12 @@ def __init__( ): import pymongo + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._database = database self._collection = collection self._dimension_separator = dimension_separator @@ -2866,6 +2923,11 @@ class RedisStore(Store): .. note:: This is an experimental feature. + .. deprecated:: 2.18.0 + RedisStore will be removed in Zarr-Python 3.0. See + `GH1274 `_ + for more information. + Requires the `redis `_ package to be installed. @@ -2885,6 +2947,12 @@ def __init__( ): import redis + warnings.warn( + V3_DEPRECATION_MESSAGE.format(store=self.__class__.__name__), + FutureWarning, + stacklevel=2, + ) + self._prefix = prefix self._kwargs = kwargs self._dimension_separator = dimension_separator From 863c3c0838adc1f65eb5fcddb6cf8fd65480acb5 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 6 May 2024 21:56:37 -0700 Subject: [PATCH 0088/1043] use np.inf instead of PINF/NINF (#1842) * use np.inf instead of PINF/NINF * update release notes --- docs/release.rst | 12 +++++------- zarr/meta.py | 4 ++-- zarr/tests/test_meta.py | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 5184ab5f9f..0cb1777210 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -25,9 +25,8 @@ Unreleased Enhancements ~~~~~~~~~~~~ -* Performance improvement for reading and writing chunks if any of the dimensions is size 1. :issue:`1730` - By :user:`Deepak Cherian `. - +* Performance improvement for reading and writing chunks if any of the dimensions is size 1. + By :user:`Deepak Cherian ` :issue:`1730`. Docs ~~~~ @@ -35,16 +34,15 @@ Docs Maintenance ~~~~~~~~~~~ +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. + By :user:`Joe Hamman ` :issue:`1842`. Deprecations ~~~~~~~~~~~~ * Deprecate experimental v3 support by issuing a `FutureWarning`. Also updated docs to warn about using the experimental v3 version. - By :user:`Joe Hamman ` :issue:`1802` and :issue: `1807`. - -Deprecations -~~~~~~~~~~~~ + By :user:`Joe Hamman ` :issue:`1802` and :issue:`1807`. * Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. diff --git a/zarr/meta.py b/zarr/meta.py index 4b360270de..747d8bec8a 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -227,9 +227,9 @@ def decode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> if v == "NaN": return np.nan elif v == "Infinity": - return np.PINF + return np.inf elif v == "-Infinity": - return np.NINF + return -np.inf else: return np.array(v, dtype=dtype)[()] elif dtype.kind in "c": diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index c85d3f923f..54347835d7 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -382,8 +382,8 @@ def test_encode_decode_array_structured(): def test_encode_decode_fill_values_nan(): fills = ( (np.nan, "NaN", np.isnan), - (np.NINF, "-Infinity", np.isneginf), - (np.PINF, "Infinity", np.isposinf), + (-np.inf, "-Infinity", np.isneginf), + (np.inf, "Infinity", np.isposinf), ) for v, s, f in fills: From 360eb53f636c33bf255682f53078e91bf6e24094 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 6 May 2024 22:40:05 -0700 Subject: [PATCH 0089/1043] chore: update pre-commit hooks (#1825) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.1 → v0.4.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.1...v0.4.3) - [github.com/psf/black: 24.4.0 → 24.4.2](https://github.com/psf/black/compare/24.4.0...24.4.2) - [github.com/pre-commit/mirrors-mypy: v1.9.0 → v1.10.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.9.0...v1.10.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0aa13b31a3..747cb86688 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,11 +8,11 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.4.1' + rev: 'v0.4.3' hooks: - id: ruff - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/codespell-project/codespell @@ -24,7 +24,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.10.0 hooks: - id: mypy files: zarr From 270aff18dc780a0a9e7409455277aae017190765 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 7 May 2024 17:29:34 +0200 Subject: [PATCH 0090/1043] Enable ruff/bugbear rules (B) and fix issues (#1702) * Enable ruff/bugbear rules (B) As suggested by Repo-Review. * Fix ruff/bugbear issue (B007) B007 Loop control variable `key` not used within loop body https://docs.astral.sh/ruff/rules/unused-loop-control-variable/ * Fix ruff/bugbear issue (B015) B015 Pointless comparison. Did you mean to assign a value? Otherwise, prepend `assert` or remove it. https://docs.astral.sh/ruff/rules/useless-comparison/ * Fix ruff/bugbear issues (B028) B028 No explicit `stacklevel` keyword argument found https://docs.astral.sh/ruff/rules/no-explicit-stacklevel/ * Fix ruff/bugbear issues (B904) B904 Within an `except` clause, raise exceptions with `raise ... from err` or `raise ... from None` to distinguish them from errors in exception handling https://docs.astral.sh/ruff/rules/raise-without-from-inside-except/ * Document changes in docs/release.rst * Disable ruff/bugbear rule (B017) B017 `pytest.raises(Exception)` should be considered evil https://docs.astral.sh/ruff/rules/assert-raises-exception/ --------- Co-authored-by: Joe Hamman --- docs/release.rst | 3 +++ pyproject.toml | 5 +++++ zarr/_storage/absstore.py | 8 ++++---- zarr/_storage/v3_storage_transformers.py | 8 ++++---- zarr/core.py | 4 ++-- zarr/creation.py | 2 +- zarr/hierarchy.py | 10 +++++----- zarr/indexing.py | 2 +- zarr/meta.py | 6 +++--- zarr/meta_v1.py | 2 +- zarr/n5.py | 22 +++++++++++++++++----- zarr/storage.py | 16 ++++++++++------ zarr/tests/test_meta.py | 2 +- zarr/tests/test_storage.py | 6 +++--- zarr/tests/test_storage_v3.py | 2 +- zarr/util.py | 12 ++++++------ 16 files changed, 67 insertions(+), 43 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 0cb1777210..e2bc40bf99 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -34,6 +34,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Enable ruff/bugbear rules (B) and fix issues. + By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. + * Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. diff --git a/pyproject.toml b/pyproject.toml index f2356480bd..dacd45ec2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -103,6 +103,11 @@ exclude = [ "docs" ] +[tool.ruff.lint] +extend-select = [ + "B" +] + [tool.black] line-length = 100 exclude = ''' diff --git a/zarr/_storage/absstore.py b/zarr/_storage/absstore.py index 5d2606f2f2..1e49754f38 100644 --- a/zarr/_storage/absstore.py +++ b/zarr/_storage/absstore.py @@ -156,8 +156,8 @@ def __getitem__(self, key): blob_name = self._append_path_to_prefix(key) try: return self.client.download_blob(blob_name).readall() - except ResourceNotFoundError: - raise KeyError(f"Blob {blob_name} not found") + except ResourceNotFoundError as e: + raise KeyError(f"Blob {blob_name} not found") from e def __setitem__(self, key, value): value = ensure_bytes(value) @@ -169,8 +169,8 @@ def __delitem__(self, key): try: self.client.delete_blob(self._append_path_to_prefix(key)) - except ResourceNotFoundError: - raise KeyError(f"Blob {key} not found") + except ResourceNotFoundError as e: + raise KeyError(f"Blob {key} not found") from e def __eq__(self, other): return ( diff --git a/zarr/_storage/v3_storage_transformers.py b/zarr/_storage/v3_storage_transformers.py index 37e56f8ecd..00467d44f9 100644 --- a/zarr/_storage/v3_storage_transformers.py +++ b/zarr/_storage/v3_storage_transformers.py @@ -183,8 +183,8 @@ def __getitem__(self, key): shard_key, chunk_subkey = self._key_to_shard(key) try: full_shard_value = self.inner_store[shard_key] - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e index = self._get_index_from_buffer(full_shard_value) chunk_slice = index.get_chunk_slice(chunk_subkey) if chunk_slice is not None: @@ -265,8 +265,8 @@ def __delitem__(self, key): shard_key, chunk_subkey = self._key_to_shard(key) try: index = self._get_index_from_store(shard_key) - except KeyError: - raise KeyError(key) + except KeyError as e: + raise KeyError(key) from e index.set_chunk_slice(chunk_subkey, None) diff --git a/zarr/core.py b/zarr/core.py index 1bd081acee..6aa86b6465 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -200,8 +200,8 @@ def _load_metadata_nosync(self): try: mkey = _prefix_to_array_key(self._store, self._key_prefix) meta_bytes = self._store[mkey] - except KeyError: - raise ArrayNotFoundError(self._path) + except KeyError as e: + raise ArrayNotFoundError(self._path) from e else: # decode and store metadata as instance members meta = self._store._metadata_class.decode_array_metadata(meta_bytes) diff --git a/zarr/creation.py b/zarr/creation.py index c541531d54..9b2b1d6d4c 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -297,7 +297,7 @@ def _kwargs_compat(compressor, fill_value, kwargs): # ignore other keyword arguments for k in kwargs: - warn(f"ignoring keyword argument {k!r}") + warn(f"ignoring keyword argument {k!r}", stacklevel=2) return compressor, fill_value diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index 0fb07dd620..8894a5ed57 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -187,16 +187,16 @@ def __init__( mkey = _prefix_to_group_key(self._store, self._key_prefix) assert not mkey.endswith("root/.group") meta_bytes = store[mkey] - except KeyError: + except KeyError as e: if self._version == 2: - raise GroupNotFoundError(path) + raise GroupNotFoundError(path) from e else: implicit_prefix = meta_root + self._key_prefix if self._store.list_prefix(implicit_prefix): # implicit group does not have any metadata self._meta = None else: - raise GroupNotFoundError(path) + raise GroupNotFoundError(path) from e else: self._meta = self._store._metadata_class.decode_group_metadata(meta_bytes) @@ -536,8 +536,8 @@ def __getattr__(self, item): # allow access to group members via dot notation try: return self.__getitem__(item) - except KeyError: - raise AttributeError + except KeyError as e: + raise AttributeError from e def __dir__(self): # noinspection PyUnresolvedReferences diff --git a/zarr/indexing.py b/zarr/indexing.py index 9889fcadad..2f2402fe27 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -932,7 +932,7 @@ def check_fields(fields, dtype): # multiple field selection out_dtype = np.dtype([(f, dtype[f]) for f in fields]) except KeyError as e: - raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") + raise IndexError(f"invalid 'fields' argument, field not found: {e!r}") from e else: return out_dtype else: diff --git a/zarr/meta.py b/zarr/meta.py index 747d8bec8a..5430ab305d 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -310,8 +310,8 @@ def decode_dtype(cls, d, validate=True): # extract the type from the extension info try: d = d["type"] - except KeyError: - raise KeyError("Extended dtype info must provide a key named 'type'.") + except KeyError as e: + raise KeyError("Extended dtype info must provide a key named 'type'.") from e d = cls._decode_dtype_descr(d) dtype = np.dtype(d) if validate: @@ -518,7 +518,7 @@ def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType meta["storage_transformers"] = storage_transformers except Exception as e: - raise MetadataError(f"error decoding metadata: {e}") + raise MetadataError(f"error decoding metadata: {e}") from e else: return meta diff --git a/zarr/meta_v1.py b/zarr/meta_v1.py index 65bfd3488e..714f55f477 100644 --- a/zarr/meta_v1.py +++ b/zarr/meta_v1.py @@ -23,7 +23,7 @@ def decode_metadata(b): order=meta["order"], ) except Exception as e: - raise MetadataError(f"error decoding metadata: {e}") + raise MetadataError(f"error decoding metadata: {e}") from e else: return meta diff --git a/zarr/n5.py b/zarr/n5.py index fdd3d5babf..3d3e9afa26 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -125,7 +125,11 @@ def __setitem__(self, key: str, value: Any): for k in n5_keywords: if k in zarr_attrs: - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) + warnings.warn( + f"Attribute {k} is a reserved N5 keyword", + UserWarning, + stacklevel=2, + ) # remove previous user attributes for k in list(n5_attrs.keys()): @@ -327,7 +331,10 @@ class N5FSStore(FSStore): def __init__(self, *args, **kwargs): if "dimension_separator" in kwargs: kwargs.pop("dimension_separator") - warnings.warn("Keyword argument `dimension_separator` will be ignored") + warnings.warn( + "Keyword argument `dimension_separator` will be ignored", + stacklevel=2, + ) dimension_separator = "." super().__init__(*args, dimension_separator=dimension_separator, **kwargs) @@ -411,7 +418,11 @@ def __setitem__(self, key: str, value: Any): for k in n5_keywords: if k in zarr_attrs.keys(): - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) + warnings.warn( + f"Attribute {k} is a reserved N5 keyword", + UserWarning, + stacklevel=2, + ) # replace previous user attributes for k in list(n5_attrs.keys()): @@ -597,8 +608,8 @@ def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dic array_metadata["n5"] = N5_FORMAT try: dtype = np.dtype(array_metadata["dataType"]) - except TypeError: - raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5") + except TypeError as e: + raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5") from e array_metadata["dataType"] = dtype.name array_metadata["dimensions"] = array_metadata["dimensions"][::-1] @@ -711,6 +722,7 @@ def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict "Not all N5 implementations support lzma compression (yet). You " "might not be able to open the dataset with another N5 library.", RuntimeWarning, + stacklevel=2, ) n5_config["format"] = _compressor_config["format"] n5_config["check"] = _compressor_config["check"] diff --git a/zarr/storage.py b/zarr/storage.py index 772fa7646a..f412870f75 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -589,11 +589,15 @@ def _init_array_metadata( "missing object_codec for object array; this will raise a " "ValueError in version 3.0", FutureWarning, + stacklevel=2, ) else: filters_config.insert(0, object_codec.get_config()) elif object_codec is not None: - warnings.warn("an object_codec is only needed for object arrays") + warnings.warn( + "an object_codec is only needed for object arrays", + stacklevel=2, + ) # use null to indicate no filters if not filters_config: @@ -869,8 +873,8 @@ def __getitem__(self, item: str): parent, key = self._get_parent(item) try: value = parent[key] - except KeyError: - raise KeyError(item) + except KeyError as e: + raise KeyError(item) from e else: if isinstance(value, self.cls): raise KeyError(item) @@ -888,8 +892,8 @@ def __delitem__(self, item: str): parent, key = self._get_parent(item) try: del parent[key] - except KeyError: - raise KeyError(item) + except KeyError as e: + raise KeyError(item) from e def __contains__(self, item: str): # type: ignore[override] try: @@ -1137,7 +1141,7 @@ def __setitem__(self, key, value): os.makedirs(dir_path) except OSError as e: if e.errno != errno.EEXIST: - raise KeyError(key) + raise KeyError(key) from e # write to temporary file # note we're not using tempfile.NamedTemporaryFile to avoid restrictive file permissions diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index 54347835d7..f9010d6788 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -601,7 +601,7 @@ def test_metadata3_exceptions(): required = ["zarr_format", "metadata_encoding", "metadata_key_suffix", "extensions"] for key in required: meta = copy.copy(_default_entry_point_metadata_v3) - meta.pop("zarr_format") + meta.pop(key) with pytest.raises(ValueError): # cannot encode metadata that is missing a required key Metadata3.encode_hierarchy_metadata(meta) diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index ae8a56fa61..da690f5959 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1353,13 +1353,13 @@ def test_exceptions(self, memory_store): # no exception from FSStore.getitems getting KeyError assert group.store.getitems(["foo"], contexts={}) == {} # exception from FSStore.getitems getting AttributeError - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 group.store.getitems(["x/0"], contexts={}) # exception from FSStore.getitems getting AttributeError - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 x[...] # exception from FSStore.__getitem__ getting AttributeError - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa: B017 y[...] diff --git a/zarr/tests/test_storage_v3.py b/zarr/tests/test_storage_v3.py index c096f9cb02..e8675786e0 100644 --- a/zarr/tests/test_storage_v3.py +++ b/zarr/tests/test_storage_v3.py @@ -286,7 +286,7 @@ def test_rename_nonexisting(self): def test_get_partial_values(self): store = self.create_store() - store.supports_efficient_get_partial_values in [True, False] + assert store.supports_efficient_get_partial_values in [True, False] store[data_root + "foo"] = b"abcdefg" store[data_root + "baz"] = b"z" assert [b"a"] == store.get_partial_values([(data_root + "foo", (0, 1))]) diff --git a/zarr/util.py b/zarr/util.py index e58aed80ab..8a96f92c24 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -198,11 +198,11 @@ def normalize_dtype(dtype: Union[str, np.dtype], object_codec) -> Tuple[np.dtype args = [] try: object_codec = codec_registry[codec_id](*args) - except KeyError: # pragma: no cover + except KeyError as e: # pragma: no cover raise ValueError( f"codec {codec_id!r} for object type {key!r} is not " f"available; please provide an object_codec manually" - ) + ) from e return dtype, object_codec dtype = np.dtype(dtype) @@ -332,7 +332,7 @@ def normalize_fill_value(fill_value, dtype: np.dtype): raise ValueError( f"fill_value {fill_value!r} is not valid for dtype {dtype}; " f"nested exception: {e}" - ) + ) from e return fill_value @@ -492,13 +492,13 @@ def tree_widget_sublist(node, root=False, expand=False): def tree_widget(group, expand, level): try: import ipytree - except ImportError as error: + except ImportError as e: raise ImportError( - f"{error}: Run `pip install zarr[jupyter]` or `conda install ipytree`" + f"{e}: Run `pip install zarr[jupyter]` or `conda install ipytree`" f"to get the required ipytree dependency for displaying the tree " f"widget. If using jupyterlab<3, you also need to run " f"`jupyter labextension install ipytree`" - ) + ) from e result = ipytree.Tree() root = TreeNode(group, level=level) From 056657ca5ed70aa3d77a9e2db42253fca39800b0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 7 May 2024 14:01:44 -0700 Subject: [PATCH 0091/1043] Release notes for 2.18.0 (#1843) * doc: cleanup release notes for 2.18.0 --- docs/release.rst | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index e2bc40bf99..ba26549402 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,20 +18,16 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. -.. _unreleased: +.. _release_2.18.0: -Unreleased ----------- +2.18.0 +------ Enhancements ~~~~~~~~~~~~ * Performance improvement for reading and writing chunks if any of the dimensions is size 1. By :user:`Deepak Cherian ` :issue:`1730`. -Docs -~~~~ - - Maintenance ~~~~~~~~~~~ * Enable ruff/bugbear rules (B) and fix issues. @@ -46,6 +42,7 @@ Deprecations * Deprecate experimental v3 support by issuing a `FutureWarning`. Also updated docs to warn about using the experimental v3 version. By :user:`Joe Hamman ` :issue:`1802` and :issue:`1807`. + * Deprecate the following stores: :class:`zarr.storage.DBMStore`, :class:`zarr.storage.LMDBStore`, :class:`zarr.storage.SQLiteStore`, :class:`zarr.storage.MongoDBStore`, :class:`zarr.storage.RedisStore`, and :class:`zarr.storage.ABSStore`. These stores are slated to be removed from Zarr-Python in version 3.0. From cb4230dd9a2a0c68b78384b24a2a5567df0b704a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 7 May 2024 21:27:39 -0700 Subject: [PATCH 0092/1043] Update release.rst (#1850) --- docs/release.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index ba26549402..e2f9f3de85 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,23 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _unreleased: + +Unreleased +---------- + +Enhancements +~~~~~~~~~~~~ + +Docs +~~~~ + +Maintenance +~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ + .. _release_2.18.0: 2.18.0 From 8979205fbebd991974172ac23b436372911f4666 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 13 May 2024 16:45:41 -0700 Subject: [PATCH 0093/1043] Group dependabot updates (#1854) --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d8e8d4d57a..5a0befe9b5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,6 +5,10 @@ updates: directory: "/" schedule: interval: "daily" + groups: + requirements: + patterns: + - "*" - package-ecosystem: "github-actions" directory: "/" schedule: From 8264acebc5e1671c36bc434f14cac914f5a099e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 20:33:05 -0700 Subject: [PATCH 0094/1043] chore: update pre-commit hooks (#1876) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.3 → v0.4.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.3...v0.4.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 747cb86688..be57770200 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ default_language_version: repos: - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: 'v0.4.3' + rev: 'v0.4.4' hooks: - id: ruff - repo: https://github.com/psf/black From 908e920bfd52371a4148b19d305262a804bc0201 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Thu, 16 May 2024 16:37:54 -0600 Subject: [PATCH 0095/1043] Fix a regression with scalar indexing due to #1800 (#1875) --- docs/release.rst | 2 ++ zarr/core.py | 4 +++- zarr/indexing.py | 2 ++ zarr/tests/test_core.py | 49 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index e2f9f3de85..a81be4638f 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -31,6 +31,8 @@ Docs Maintenance ~~~~~~~~~~~ +* Fix a regression when getting or setting a single value from arrays with size-1 chunks. + By :user:`Deepak Cherian ` :issue:`1874` Deprecations ~~~~~~~~~~~~ diff --git a/zarr/core.py b/zarr/core.py index 6aa86b6465..b1ccd203db 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2030,7 +2030,9 @@ def _process_chunk( and not self._filters and self._dtype != object ): - dest = out[out_selection] + # For 0D arrays out_selection = () and out[out_selection] is a scalar + # Avoid that + dest = out[out_selection] if out_selection else out # Assume that array-like objects that doesn't have a # `writeable` flag is writable. dest_is_writable = getattr(dest, "writeable", True) diff --git a/zarr/indexing.py b/zarr/indexing.py index 2f2402fe27..35c1e813b1 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -52,6 +52,8 @@ def is_scalar(value, dtype): return True if isinstance(value, tuple) and dtype.names and len(value) == len(dtype.names): return True + if dtype.kind == "O" and not isinstance(value, np.ndarray): + return True return False diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 730f724314..01a78ecd68 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3157,3 +3157,52 @@ def test_issue_1279(tmpdir): written_data = ds_reopened[:] assert_array_equal(data, written_data) + + +def test_scalar_indexing(): + store = zarr.KVStore({}) + + store["a"] = zarr.create((3,), chunks=(1,), store=store) + store["a"][:] = [1, 2, 3] + + assert store["a"][1] == np.array(2.0) + assert store["a"][(1,)] == np.array(2.0) + + store["a"][slice(1)] = [-1] + assert store["a"][0] == np.array(-1) + + store["a"][0] = -2 + assert store["a"][0] == np.array(-2) + + store["a"][slice(1)] = (-3,) + assert store["a"][0] == np.array(-3) + + +def test_object_array_indexing(): + # regression test for #1874 + from numcodecs import MsgPack + + root = zarr.group() + arr = root.create_dataset( + name="my_dataset", + shape=0, + dtype=object, + object_codec=MsgPack(), + ) + new_items = [ + ["A", 1], + ["B", 2, "hello"], + ] + arr_add = np.empty(len(new_items), dtype=object) + arr_add[:] = new_items + arr.append(arr_add) + + # heterogeneous elements + elem = ["C", 3] + arr[0] = elem + assert arr[0] == elem + + # homogeneous elements + elem = [1, 3] + arr[1] = elem + assert arr[1] == elem From 4bd9764c1857cb563d155ad797011dfd4a44616e Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 16 May 2024 17:57:40 -0700 Subject: [PATCH 0096/1043] release notes for 2.18.1 (#1885) --- docs/release.rst | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index a81be4638f..5ca60b8166 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,25 +18,16 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. -.. _unreleased: +.. _release_2.18.1: -Unreleased ----------- - -Enhancements -~~~~~~~~~~~~ - -Docs -~~~~ +2.18.1 +------ Maintenance ~~~~~~~~~~~ * Fix a regression when getting or setting a single value from arrays with size-1 chunks. By :user:`Deepak Cherian ` :issue:`1874` -Deprecations -~~~~~~~~~~~~ - .. _release_2.18.0: 2.18.0 From bf895332cdb8f028284f958c58cda773085d5e68 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 17 May 2024 05:52:31 -0700 Subject: [PATCH 0097/1043] reset release notes (#1886) --- docs/release.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 5ca60b8166..59051bbf97 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,23 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _unreleased: + +Unreleased +---------- + +Enhancements +~~~~~~~~~~~~ + +Docs +~~~~ + +Maintenance +~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ + .. _release_2.18.1: 2.18.1 From 3e98c3b1d6ed24b7b04fdbaa56ed2c42f5b1ef05 Mon Sep 17 00:00:00 2001 From: Ryan Abernathey Date: Sun, 26 May 2024 14:54:33 -0400 Subject: [PATCH 0098/1043] Add zstd to old V3 supported codecs (#1914) * add zstd to old V3 supported codecs * get to full test coverage and add release note * fix pre-commit --- docs/release.rst | 3 +++ zarr/codecs.py | 2 +- zarr/meta.py | 4 ++++ zarr/tests/test_meta.py | 23 +++++++++++++++++------ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 59051bbf97..48a6c9ca2d 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -32,6 +32,9 @@ Docs Maintenance ~~~~~~~~~~~ +* Add Zstd codec to old V3 code path. + By :user:`Ryan Abernathey ` + Deprecations ~~~~~~~~~~~~ diff --git a/zarr/codecs.py b/zarr/codecs.py index 4ad68b8627..6fd5e20401 100644 --- a/zarr/codecs.py +++ b/zarr/codecs.py @@ -1,4 +1,4 @@ # flake8: noqa from numcodecs import * -from numcodecs import get_codec, Blosc, Pickle, Zlib, Delta, AsType, BZ2 +from numcodecs import get_codec, Blosc, Pickle, Zlib, Zstd, Delta, AsType, BZ2 from numcodecs.registry import codec_registry diff --git a/zarr/meta.py b/zarr/meta.py index 5430ab305d..44a2b7ebec 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -414,6 +414,8 @@ def _encode_codec_metadata(cls, codec: Codec) -> Optional[Mapping]: uri = uri + "lz4/1.0" elif isinstance(codec, numcodecs.LZMA): uri = uri + "lzma/1.0" + elif isinstance(codec, numcodecs.Zstd): + uri = uri + "zstd/1.0" meta = { "codec": uri, "configuration": config, @@ -439,6 +441,8 @@ def _decode_codec_metadata(cls, meta: Optional[Mapping]) -> Optional[Codec]: conf["id"] = "lz4" elif meta["codec"].startswith(uri + "lzma/"): conf["id"] = "lzma" + elif meta["codec"].startswith(uri + "zstd/"): + conf["id"] = "zstd" else: raise NotImplementedError diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index f9010d6788..7b7d526476 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -5,7 +5,7 @@ import numpy as np import pytest -from zarr.codecs import Blosc, Delta, Pickle, Zlib +from zarr.codecs import Blosc, Delta, Pickle, Zlib, Zstd from zarr.errors import MetadataError from zarr.meta import ( ZARR_FORMAT, @@ -268,17 +268,23 @@ def test_encode_decode_array_dtype_shape(): assert meta_dec["filters"] is None -def test_encode_decode_array_dtype_shape_v3(): +@pytest.mark.parametrize("cname", ["zlib", "zstd"]) +def test_encode_decode_array_dtype_shape_v3(cname): + if cname == "zlib": + compressor = Zlib(1) + elif cname == "zstd": + compressor = Zstd(1) meta = dict( shape=(100,), chunk_grid=dict(type="regular", chunk_shape=(10,), separator=("/")), data_type=np.dtype("(10, 10) Date: Sun, 26 May 2024 12:20:32 -0700 Subject: [PATCH 0099/1043] doc: update release notes for 2.18.2 (#1915) --- docs/release.rst | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 48a6c9ca2d..6c7ba5139b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,26 +18,14 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. -.. _unreleased: - -Unreleased ----------- +.. _release_2.18.2: Enhancements ~~~~~~~~~~~~ -Docs -~~~~ - -Maintenance -~~~~~~~~~~~ - * Add Zstd codec to old V3 code path. By :user:`Ryan Abernathey ` -Deprecations -~~~~~~~~~~~~ - .. _release_2.18.1: 2.18.1 From 5cf2ef102da7189d0bb966cfa7b25075475ad5e0 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 9 Aug 2024 22:18:34 +0100 Subject: [PATCH 0100/1043] Fix low contrast in dark theme h3 headings (#2003) --- docs/_static/custom.css | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 87dd70e347..d7b2435335 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -117,7 +117,3 @@ html[data-theme=dark] .sd-card .sd-card-footer { html[data-theme=dark] h1 { color: var(--pst-color-primary); } - -html[data-theme=dark] h3 { - color: #0a6774; -} From f6de8846cc992d355dcbd55f3a8421529b033b78 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 9 Aug 2024 16:22:43 -0500 Subject: [PATCH 0101/1043] Fixed dependencies (#2069) - removed pytest as a required dep - move test deps to a "test" optional dependency --- pyproject.toml | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 93116bede8..88542b2cf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,6 @@ dependencies = [ 'zstandard', 'typing_extensions', 'donfig', - 'pytest' ] dynamic = [ "version", @@ -55,6 +54,22 @@ license = {text = "MIT License"} keywords = ["Python", "compressed", "ndimensional-arrays", "zarr"] [project.optional-dependencies] +test = [ + "coverage", + "pytest", + "pytest-cov", + "msgpack", + "lmdb", + "s3fs", + "pytest-asyncio", + "moto[s3]", + "flask-cors", + "flask", + "requests", + "mypy", + "hypothesis" +] + jupyter = [ 'notebook', 'ipytree>=0.2.2', @@ -108,22 +123,7 @@ dependencies = [ "numpy~={matrix:numpy}", "universal_pathlib" ] -extra-dependencies = [ - "coverage", - "pytest", - "pytest-cov", - "msgpack", - "lmdb", - "s3fs", - "pytest-asyncio", - "moto[s3]", - "flask-cors", - "flask", - "requests", - "mypy", - "hypothesis" -] -features = ["extra"] +features = ["test", "extra"] [[tool.hatch.envs.test.matrix]] python = ["3.10", "3.11", "3.12"] From a054afbbdaf7c8560d0ce5df77af969bc863e335 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 10 Aug 2024 09:16:40 +0100 Subject: [PATCH 0102/1043] Drop support for Python 3.9 --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/releases.yml | 2 +- .readthedocs.yaml | 2 +- docs/release.rst | 10 ++++++++-- pyproject.toml | 4 ++-- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ec98af029e..7b0c4dcfc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.9, 3.10, 3.11, etc. + placeholder: 3.10, 3.11, 3.12 etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f53cb2d9a9..61124b8e8e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 8ac76c899b..f94711ad63 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.1.0 name: Install Python with: - python-version: '3.9' + python-version: '3.11' - name: Install PyBuild run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e45cae1b45..d7190b4771 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py diff --git a/docs/release.rst b/docs/release.rst index 6c7ba5139b..703fec8df6 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -24,7 +24,13 @@ Enhancements ~~~~~~~~~~~~ * Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` + By :user:`Ryan Abernathey ` + +Maintenance +~~~~~~~~~~~ + +* Removed support for Python 3.9. + By :user:`David Stansby ` .. _release_2.18.1: @@ -51,7 +57,7 @@ Maintenance * Enable ruff/bugbear rules (B) and fix issues. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. Deprecations diff --git a/pyproject.toml b/pyproject.toml index dacd45ec2c..a0525a8d8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ 'asciitree', 'numpy>=1.23', @@ -30,9 +30,9 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ] license = { text = "MIT" } From 5437386047323657a7d3ecc5bf775352ef30f800 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 10 Aug 2024 09:19:06 +0100 Subject: [PATCH 0103/1043] Revert "Drop support for Python 3.9" This reverts commit a054afbbdaf7c8560d0ce5df77af969bc863e335. --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/releases.yml | 2 +- .readthedocs.yaml | 2 +- docs/release.rst | 10 ++-------- pyproject.toml | 4 ++-- 6 files changed, 8 insertions(+), 14 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 7b0c4dcfc4..ec98af029e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.10, 3.11, 3.12 etc. + placeholder: 3.9, 3.10, 3.11, etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 61124b8e8e..f53cb2d9a9 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index f94711ad63..8ac76c899b 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.1.0 name: Install Python with: - python-version: '3.11' + python-version: '3.9' - name: Install PyBuild run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index d7190b4771..e45cae1b45 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.11" + python: "3.9" sphinx: configuration: docs/conf.py diff --git a/docs/release.rst b/docs/release.rst index 703fec8df6..6c7ba5139b 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -24,13 +24,7 @@ Enhancements ~~~~~~~~~~~~ * Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` - -Maintenance -~~~~~~~~~~~ - -* Removed support for Python 3.9. - By :user:`David Stansby ` + By :user:`Ryan Abernathey ` .. _release_2.18.1: @@ -57,7 +51,7 @@ Maintenance * Enable ruff/bugbear rules (B) and fix issues. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. Deprecations diff --git a/pyproject.toml b/pyproject.toml index a0525a8d8a..dacd45ec2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.10" +requires-python = ">=3.9" dependencies = [ 'asciitree', 'numpy>=1.23', @@ -30,9 +30,9 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', - 'Programming Language :: Python :: 3.12', ] license = { text = "MIT" } From d9a142a5f677bb0289d97119891d03a20032600d Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Sun, 11 Aug 2024 13:25:56 +0200 Subject: [PATCH 0104/1043] Remove zstandard dependency in favor of numcodecs (#1838) * removes zstandard dependency * fix dependencies * change default level to 3 * revert to 0 default level --- .pre-commit-config.yaml | 1 - pyproject.toml | 5 ++--- src/zarr/codecs/zstd.py | 30 ++++++++++++++++++------------ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8abb9fddd3..67137be96c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,6 @@ repos: - numcodecs - numpy - typing_extensions - - zstandard # Tests - pytest # Zarr v2 diff --git a/pyproject.toml b/pyproject.toml index 88542b2cf5..53b4cb3244 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ dependencies = [ 'numcodecs>=0.10.0', 'fsspec>2024', 'crc32c', - 'zstandard', 'typing_extensions', 'donfig', ] @@ -85,8 +84,8 @@ docs = [ 'pydata-sphinx-theme', 'numpydoc', 'numcodecs[msgpack]', - "msgpack", - "lmdb", + 'msgpack', + 'lmdb', ] extra = [ 'msgpack', diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index b244ee703a..3b3d3f33dd 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -1,10 +1,11 @@ from __future__ import annotations from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from functools import cached_property +from importlib.metadata import version +from typing import TYPE_CHECKING -import numpy.typing as npt -from zstandard import ZstdCompressor, ZstdDecompressor +from numcodecs.zstd import Zstd from zarr.abc.codec import BytesBytesCodec from zarr.array_spec import ArraySpec @@ -38,6 +39,14 @@ class ZstdCodec(BytesBytesCodec): checksum: bool = False def __init__(self, *, level: int = 0, checksum: bool = False) -> None: + # numcodecs 0.13.0 introduces the checksum attribute for the zstd codec + _numcodecs_version = tuple(map(int, version("numcodecs").split("."))) + if _numcodecs_version < (0, 13, 0): # pragma: no cover + raise RuntimeError( + "numcodecs version >= 0.13.0 is required to use the zstd codec. " + f"Version {_numcodecs_version} is currently installed." + ) + level_parsed = parse_zstd_level(level) checksum_parsed = parse_checksum(checksum) @@ -52,13 +61,10 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: def to_dict(self) -> dict[str, JSON]: return {"name": "zstd", "configuration": {"level": self.level, "checksum": self.checksum}} - def _compress(self, data: npt.NDArray[Any]) -> bytes: - ctx = ZstdCompressor(level=self.level, write_checksum=self.checksum) - return ctx.compress(data.tobytes()) - - def _decompress(self, data: npt.NDArray[Any]) -> bytes: - ctx = ZstdDecompressor() - return ctx.decompress(data.tobytes()) + @cached_property + def _zstd_codec(self) -> Zstd: + config_dict = {"level": self.level, "checksum": self.checksum} + return Zstd.from_config(config_dict) async def _decode_single( self, @@ -66,7 +72,7 @@ async def _decode_single( chunk_spec: ArraySpec, ) -> Buffer: return await to_thread( - as_numpy_array_wrapper, self._decompress, chunk_bytes, chunk_spec.prototype + as_numpy_array_wrapper, self._zstd_codec.decode, chunk_bytes, chunk_spec.prototype ) async def _encode_single( @@ -75,7 +81,7 @@ async def _encode_single( chunk_spec: ArraySpec, ) -> Buffer | None: return await to_thread( - as_numpy_array_wrapper, self._compress, chunk_bytes, chunk_spec.prototype + as_numpy_array_wrapper, self._zstd_codec.encode, chunk_bytes, chunk_spec.prototype ) def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) -> int: From 6888c593a74eedf0d153e763005754e68955bd9f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sun, 11 Aug 2024 04:42:44 -0700 Subject: [PATCH 0105/1043] Update TEAM.md (#2071) --- TEAM.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TEAM.md b/TEAM.md index 6a22d83d1f..824e57fa7f 100644 --- a/TEAM.md +++ b/TEAM.md @@ -7,6 +7,7 @@ - @jakirkham (jakirkham) - @martindurant (Martin Durant) - @normanrz (Norman Rzepka) +- @dstansby (David Stansby) ## Emeritus core-developers - @alimanfoo (Alistair Miles) From ff462cca40cbde9f2e8fe648272d42cca9b91bb8 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Sun, 11 Aug 2024 14:12:03 +0200 Subject: [PATCH 0106/1043] Enforce ruff/tryceratops rules (TRY) (#2054) * Apply ruff/tryceratops rule TRY004 TRY004 Prefer `TypeError` exception for invalid type * Apply ruff/tryceratops rule TRY201 TRY201 Use `raise` without specifying exception name * Apply ruff/tryceratops rule TRY300 TRY300 Consider moving this statement to an `else` block * Enforce ruff/tryceratops rules (TRY) --------- Co-authored-by: David Stansby --- pyproject.toml | 2 ++ src/zarr/api/asynchronous.py | 6 +++--- src/zarr/array.py | 4 ++-- src/zarr/codecs/pipeline.py | 6 +++--- src/zarr/codecs/sharding.py | 2 +- src/zarr/group.py | 3 ++- src/zarr/store/core.py | 3 ++- src/zarr/store/local.py | 3 ++- src/zarr/store/remote.py | 3 ++- tests/v3/test_codecs/test_codecs.py | 2 +- 10 files changed, 20 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 53b4cb3244..cf5c242406 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -182,9 +182,11 @@ extend-select = [ "UP", # pyupgrade "RSE", "RUF", + "TRY", # tryceratops ] ignore = [ "RUF005", + "TRY003", ] [tool.mypy] diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 5d2e54baa3..a7171b4915 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -66,7 +66,7 @@ def _like_args(a: ArrayLike, kwargs: dict[str, Any]) -> dict[str, Any]: if isinstance(a.metadata, ArrayV3Metadata): new["codecs"] = a.metadata.codecs else: - raise ValueError(f"Unsupported zarr format: {a.metadata.zarr_format}") + raise TypeError(f"Unsupported zarr format: {a.metadata.zarr_format}") else: # TODO: set default values compressor/codecs # to do this, we may need to evaluate if this is a v2 or v3 array @@ -862,7 +862,7 @@ async def open_array( try: return await AsyncArray.open(store_path, zarr_format=zarr_format) - except FileNotFoundError as e: + except FileNotFoundError: if store_path.store.mode.create: return await create( store=store_path, @@ -871,7 +871,7 @@ async def open_array( overwrite=store_path.store.mode.overwrite, **kwargs, ) - raise e + raise async def open_like(a: ArrayLike, path: str, **kwargs: Any) -> AsyncArray: diff --git a/src/zarr/array.py b/src/zarr/array.py index e41118805e..7aaf9e15ba 100644 --- a/src/zarr/array.py +++ b/src/zarr/array.py @@ -87,7 +87,7 @@ def create_codec_pipeline(metadata: ArrayV2Metadata | ArrayV3Metadata) -> CodecP [V2Filters(metadata.filters or []), V2Compressor(metadata.compressor)] ) else: - raise AssertionError + raise TypeError @dataclass(frozen=True) @@ -394,7 +394,7 @@ def chunks(self) -> ChunkCoords: if isinstance(self.metadata.chunk_grid, RegularChunkGrid): return self.metadata.chunk_grid.chunk_shape else: - raise ValueError( + raise TypeError( f"chunk attribute is only available for RegularChunkGrid, this array has a {self.metadata.chunk_grid}" ) diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/codecs/pipeline.py index 8cda04c9ff..1582cd6adb 100644 --- a/src/zarr/codecs/pipeline.py +++ b/src/zarr/codecs/pipeline.py @@ -492,7 +492,7 @@ def codecs_from_list( "must be preceded by another ArrayArrayCodec. " f"Got {type(prev_codec)} instead." ) - raise ValueError(msg) + raise TypeError(msg) array_array += (cur_codec,) elif isinstance(cur_codec, ArrayBytesCodec): @@ -501,7 +501,7 @@ def codecs_from_list( f"Invalid codec order. ArrayBytes codec {cur_codec}" f" must be preceded by an ArrayArrayCodec. Got {type(prev_codec)} instead." ) - raise ValueError(msg) + raise TypeError(msg) if array_bytes_maybe is not None: msg = ( @@ -521,7 +521,7 @@ def codecs_from_list( ) bytes_bytes += (cur_codec,) else: - raise AssertionError + raise TypeError if array_bytes_maybe is None: raise ValueError("Required ArrayBytesCodec was not found.") diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index ef8b80c02d..be236beff1 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -387,7 +387,7 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun "The shard's `chunk_shape` and array's `shape` need to have the same number of dimensions." ) if not isinstance(chunk_grid, RegularChunkGrid): - raise ValueError("Sharding is only compatible with regular chunk grids.") + raise TypeError("Sharding is only compatible with regular chunk grids.") if not all( s % c == 0 for s, c in zip( diff --git a/src/zarr/group.py b/src/zarr/group.py index 432ebc4604..a02fa6196a 100644 --- a/src/zarr/group.py +++ b/src/zarr/group.py @@ -467,9 +467,10 @@ async def contains(self, member: str) -> bool: # TODO: this can be made more efficient. try: await self.getitem(member) - return True except KeyError: return False + else: + return True # todo: decide if this method should be separate from `groups` async def group_keys(self) -> AsyncGenerator[str, None]: diff --git a/src/zarr/store/core.py b/src/zarr/store/core.py index fa35879308..4d31118a54 100644 --- a/src/zarr/store/core.py +++ b/src/zarr/store/core.py @@ -251,9 +251,10 @@ async def contains_group(store_path: StorePath, zarr_format: ZarrFormat) -> bool extant_meta_json = json.loads(extant_meta_bytes.to_bytes()) # we avoid constructing a full metadata document here in the name of speed. result: bool = extant_meta_json["node_type"] == "group" - return result except (ValueError, KeyError): return False + else: + return result elif zarr_format == 2: return await (store_path / ZGROUP_JSON).exists() msg = f"Invalid zarr_format provided. Got {zarr_format}, expected 2 or 3" # type: ignore[unreachable] diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index 25fd9fc13a..3cc42c6c0e 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -87,9 +87,10 @@ async def clear(self) -> None: async def empty(self) -> bool: try: subpaths = os.listdir(self.root) - return not subpaths except FileNotFoundError: return True + else: + return not subpaths def __str__(self) -> str: return f"file://{self.root}" diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index c742d9e567..84f01c7852 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -118,7 +118,6 @@ async def get( else self._fs._cat_file(path) ) ) - return value except self.allowed_exceptions: return None @@ -127,6 +126,8 @@ async def get( # this is an s3-specific condition we probably don't want to leak return prototype.buffer.from_bytes(b"") raise + else: + return value async def set( self, diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/v3/test_codecs/test_codecs.py index a2b459f60d..2a1b99c42f 100644 --- a/tests/v3/test_codecs/test_codecs.py +++ b/tests/v3/test_codecs/test_codecs.py @@ -382,7 +382,7 @@ def test_invalid_metadata(store: Store) -> None: fill_value=0, ) spath2 = StorePath(store, "invalid_endian") - with pytest.raises(ValueError): + with pytest.raises(TypeError): Array.create( spath2, shape=(16, 16), From 035e2f843e56b9ce19136e5e70569678b23d9958 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 14:21:19 -0700 Subject: [PATCH 0107/1043] chore: update pre-commit hooks (#2080) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.6 → v0.5.7](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.6...v0.5.7) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 67137be96c..d17b765548 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.5.6' + rev: 'v0.5.7' hooks: - id: ruff args: ["--fix", "--show-fixes"] From c72d0e2a14caffcde2035e182eb4235ecbdfee1c Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 14 Aug 2024 08:46:38 +0100 Subject: [PATCH 0108/1043] [v2] Fix doctests with numpy 2.0 (#2073) --- pyproject.toml | 4 ++++ zarr/core.py | 22 +++++++++++----------- zarr/creation.py | 2 +- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dacd45ec2c..a5668d0d90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,6 +145,10 @@ filterwarnings = [ "ignore:The .* is deprecated and will be removed in a Zarr-Python version 3*:FutureWarning", "ignore:The experimental Zarr V3 implementation in this version .*:FutureWarning", ] +doctest_subpackage_requires =[ + "zarr/core.py = numpy>=2", + "zarr/creation.py = numpy>=2" +] [tool.codespell] diff --git a/zarr/core.py b/zarr/core.py index b1ccd203db..4f3080e46c 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -609,11 +609,11 @@ def islice(self, start=None, end=None): Iterate over part of the array: >>> for value in z.islice(25, 30): value; - 25 - 26 - 27 - 28 - 29 + np.int64(25) + np.int64(26) + np.int64(27) + np.int64(28) + np.int64(29) """ if len(self.shape) == 0: @@ -679,7 +679,7 @@ def __getitem__(self, selection): Retrieve a single item:: >>> z[5] - 5 + np.int64(5) Retrieve a region via slicing:: @@ -706,7 +706,7 @@ def __getitem__(self, selection): Retrieve an item:: >>> z[2, 2] - 22 + np.int64(22) Retrieve a region via slicing:: @@ -830,7 +830,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): Retrieve a single item:: >>> z.get_basic_selection(5) - 5 + np.int64(5) Retrieve a region via slicing:: @@ -852,7 +852,7 @@ def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): Retrieve an item:: >>> z.get_basic_selection((2, 2)) - 22 + np.int64(22) Retrieve a region via slicing:: @@ -2819,7 +2819,7 @@ def view( >>> v[:] array([False, False, True, ..., True, False, False]) >>> np.all(a[:].view(dtype=bool) == v[:]) - True + np.True_ An array can be viewed with a dtype with a different item size, however some care is needed to adjust the shape and chunk shape so that chunk @@ -2833,7 +2833,7 @@ def view( >>> v[:10] array([0, 0, 1, 0, 2, 0, 3, 0, 4, 0], dtype=uint8) >>> np.all(a[:].view('u1') == v[:]) - True + np.True_ Change fill value for uninitialized chunks: diff --git a/zarr/creation.py b/zarr/creation.py index 9b2b1d6d4c..f7f3d5a094 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -569,7 +569,7 @@ def open_array( >>> z2 >>> np.all(z1[:] == z2[:]) - True + np.True_ Notes ----- From f231bb2bbbbbc96bfdf3afdb56acb66baeff9699 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 14 Aug 2024 09:18:51 +0100 Subject: [PATCH 0109/1043] Fix version number in built docs (#2044) --- docs/conf.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 048e77f51d..886160382f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,11 +16,8 @@ import os import sys -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. +from importlib.metadata import version as get_version + import zarr # If extensions (or modules to document with autodoc) are in another directory, @@ -75,9 +72,8 @@ copyright = "2024, Zarr Developers" author = "Zarr Developers" -version = zarr.__version__ -# The full version, including alpha/beta/rc tags. -release = zarr.__version__ +version = get_version("zarr") +release = get_version("zarr") # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 10ae5f3b1e2b67269c2181617302f305ef4b277c Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 14 Aug 2024 03:00:01 -0700 Subject: [PATCH 0110/1043] [v3] reorganize package - move most modules to zarr.core (#2072) * create core subpackage and move initial modules * rev 2 * lint * rev 3 * move __all__s --- src/zarr/__init__.py | 6 +-- src/zarr/abc/codec.py | 35 +++++++++++------ src/zarr/abc/metadata.py | 4 +- src/zarr/abc/store.py | 6 ++- src/zarr/api/asynchronous.py | 40 +++++++++++++++++--- src/zarr/api/synchronous.py | 38 ++++++++++++++++--- src/zarr/codecs/_v2.py | 6 +-- src/zarr/codecs/blosc.py | 6 +-- src/zarr/codecs/bytes.py | 6 +-- src/zarr/codecs/crc32c_.py | 6 +-- src/zarr/codecs/gzip.py | 6 +-- src/zarr/codecs/pipeline.py | 12 +++--- src/zarr/codecs/sharding.py | 24 +++++++----- src/zarr/codecs/transpose.py | 8 ++-- src/zarr/codecs/zstd.py | 6 +-- src/zarr/convenience.py | 20 +++++----- src/zarr/core/__init__.py | 0 src/zarr/{ => core}/array.py | 32 +++++++--------- src/zarr/{ => core}/array_spec.py | 9 +++-- src/zarr/{ => core}/attributes.py | 10 +++-- src/zarr/{ => core}/buffer.py | 4 +- src/zarr/{ => core}/chunk_grids.py | 4 +- src/zarr/{ => core}/chunk_key_encodings.py | 2 +- src/zarr/{ => core}/common.py | 0 src/zarr/{ => core}/config.py | 4 +- src/zarr/{ => core}/group.py | 18 +++++---- src/zarr/{ => core}/indexing.py | 11 +++--- src/zarr/{ => core}/metadata.py | 13 +++---- src/zarr/{ => core}/sync.py | 13 +++---- src/zarr/creation.py | 12 +++--- src/zarr/errors.py | 7 ++++ src/zarr/registry.py | 24 +++++++++--- src/zarr/store/__init__.py | 2 +- src/zarr/store/{utils.py => _utils.py} | 2 +- src/zarr/store/{core.py => common.py} | 10 +++-- src/zarr/store/local.py | 9 ++++- src/zarr/store/memory.py | 11 ++++-- src/zarr/store/remote.py | 9 ++--- src/zarr/strategies.py | 5 +-- src/zarr/testing/__init__.py | 6 ++- src/zarr/testing/buffer.py | 9 ++++- src/zarr/testing/store.py | 7 +++- src/zarr/testing/utils.py | 6 ++- tests/v3/conftest.py | 5 +-- tests/v3/package_with_entrypoint/__init__.py | 6 +-- tests/v3/test_array.py | 7 ++-- tests/v3/test_buffer.py | 6 +-- tests/v3/test_chunk_grids.py | 2 +- tests/v3/test_codecs/test_blosc.py | 6 +-- tests/v3/test_codecs/test_codecs.py | 9 ++--- tests/v3/test_codecs/test_endian.py | 6 +-- tests/v3/test_codecs/test_gzip.py | 4 +- tests/v3/test_codecs/test_sharding.py | 6 +-- tests/v3/test_codecs/test_transpose.py | 9 ++--- tests/v3/test_codecs/test_zstd.py | 4 +- tests/v3/test_common.py | 4 +- tests/v3/test_config.py | 12 +++--- tests/v3/test_group.py | 12 +++--- tests/v3/test_indexing.py | 8 ++-- tests/v3/test_metadata/test_v2.py | 2 +- tests/v3/test_metadata/test_v3.py | 8 ++-- tests/v3/test_store/test_core.py | 2 +- tests/v3/test_store/test_local.py | 2 +- tests/v3/test_store/test_memory.py | 2 +- tests/v3/test_store/test_remote.py | 4 +- tests/v3/test_sync.py | 2 +- tests/v3/test_v2.py | 2 +- 67 files changed, 360 insertions(+), 238 deletions(-) create mode 100644 src/zarr/core/__init__.py rename src/zarr/{ => core}/array.py (98%) rename src/zarr/{ => core}/array_spec.py (80%) rename src/zarr/{ => core}/attributes.py (82%) rename src/zarr/{ => core}/buffer.py (99%) rename src/zarr/{ => core}/chunk_grids.py (98%) rename src/zarr/{ => core}/chunk_key_encodings.py (98%) rename src/zarr/{ => core}/common.py (100%) rename src/zarr/{ => core}/config.py (96%) rename src/zarr/{ => core}/group.py (98%) rename src/zarr/{ => core}/indexing.py (99%) rename src/zarr/{ => core}/metadata.py (98%) rename src/zarr/{ => core}/sync.py (99%) rename src/zarr/store/{utils.py => _utils.py} (94%) rename src/zarr/store/{core.py => common.py} (96%) diff --git a/src/zarr/__init__.py b/src/zarr/__init__.py index 227b0cf63e..0259672f74 100644 --- a/src/zarr/__init__.py +++ b/src/zarr/__init__.py @@ -26,9 +26,9 @@ zeros, zeros_like, ) -from zarr.array import Array, AsyncArray -from zarr.config import config -from zarr.group import AsyncGroup, Group +from zarr.core.array import Array, AsyncArray +from zarr.core.config import config +from zarr.core.group import AsyncGroup, Group # in case setuptools scm screw up and find version to be 0.0.0 assert not __version__.startswith("0.0.0") diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index 9223019fab..fa76399f90 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -8,17 +8,28 @@ from zarr.abc.metadata import Metadata from zarr.abc.store import ByteGetter, ByteSetter -from zarr.buffer import Buffer, NDBuffer -from zarr.chunk_grids import ChunkGrid -from zarr.common import ChunkCoords, concurrent_map -from zarr.config import config +from zarr.core.buffer import Buffer, NDBuffer +from zarr.core.chunk_grids import ChunkGrid +from zarr.core.common import ChunkCoords, concurrent_map +from zarr.core.config import config if TYPE_CHECKING: from typing_extensions import Self - from zarr.array_spec import ArraySpec - from zarr.common import JSON - from zarr.indexing import SelectorTuple + from zarr.core.array_spec import ArraySpec + from zarr.core.common import JSON + from zarr.core.indexing import SelectorTuple + +__all__ = [ + "CodecInput", + "CodecOutput", + "ArrayArrayCodec", + "ArrayBytesCodec", + "BytesBytesCodec", + "ArrayBytesCodecPartialDecodeMixin", + "ArrayBytesCodecPartialEncodeMixin", + "CodecPipeline", +] CodecInput = TypeVar("CodecInput", bound=NDBuffer | Buffer) CodecOutput = TypeVar("CodecOutput", bound=NDBuffer | Buffer) @@ -112,7 +123,7 @@ async def decode( ------- Iterable[CodecInput | None] """ - return await batching_helper(self._decode_single, chunks_and_specs) + return await _batching_helper(self._decode_single, chunks_and_specs) async def _encode_single( self, chunk_data: CodecInput, chunk_spec: ArraySpec @@ -135,7 +146,7 @@ async def encode( ------- Iterable[CodecOutput | None] """ - return await batching_helper(self._encode_single, chunks_and_specs) + return await _batching_helper(self._encode_single, chunks_and_specs) class ArrayArrayCodec(_Codec[NDBuffer, NDBuffer]): @@ -401,18 +412,18 @@ def from_dict(cls, data: Iterable[JSON | Codec]) -> Self: return cls(**data) -async def batching_helper( +async def _batching_helper( func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]], batch_info: Iterable[tuple[CodecInput | None, ArraySpec]], ) -> list[CodecOutput | None]: return await concurrent_map( list(batch_info), - noop_for_none(func), + _noop_for_none(func), config.get("async.concurrency"), ) -def noop_for_none( +def _noop_for_none( func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]], ) -> Callable[[CodecInput | None, ArraySpec], Awaitable[CodecOutput | None]]: async def wrap(chunk: CodecInput | None, chunk_spec: ArraySpec) -> CodecOutput | None: diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py index a14ffb8ce9..0ddf762766 100644 --- a/src/zarr/abc/metadata.py +++ b/src/zarr/abc/metadata.py @@ -8,7 +8,9 @@ from dataclasses import dataclass, fields -from zarr.common import JSON +from zarr.core.common import JSON + +__all__ = ["Metadata"] @dataclass(frozen=True) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 449816209b..00b81a3aad 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -4,8 +4,10 @@ from typing_extensions import Self -from zarr.buffer import Buffer, BufferPrototype -from zarr.common import AccessModeLiteral, BytesLike +from zarr.core.buffer import Buffer, BufferPrototype +from zarr.core.common import AccessModeLiteral, BytesLike + +__all__ = ["Store", "AccessMode", "ByteGetter", "ByteSetter", "set_or_delete"] class AccessMode(NamedTuple): diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index a7171b4915..ad89584b44 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -9,17 +9,45 @@ import numpy.typing as npt from zarr.abc.codec import Codec -from zarr.array import Array, AsyncArray -from zarr.buffer import NDArrayLike -from zarr.chunk_key_encodings import ChunkKeyEncoding -from zarr.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat -from zarr.group import AsyncGroup -from zarr.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.array import Array, AsyncArray +from zarr.core.buffer import NDArrayLike +from zarr.core.chunk_key_encodings import ChunkKeyEncoding +from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat +from zarr.core.group import AsyncGroup +from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata from zarr.store import ( StoreLike, make_store_path, ) +__all__ = [ + "consolidate_metadata", + "copy", + "copy_all", + "copy_store", + "load", + "open", + "open_consolidated", + "save", + "save_array", + "save_group", + "tree", + "array", + "group", + "open_group", + "create", + "empty", + "empty_like", + "full", + "full_like", + "ones", + "ones_like", + "open_array", + "open_like", + "zeros", + "zeros_like", +] + # TODO: this type could use some more thought, noqa to avoid "Variable "asynchronous.ArrayLike" is not valid as a type" ArrayLike = Union[AsyncArray | Array | npt.NDArray[Any]] # noqa PathLike = str diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index eef87aab7e..d3d674c3fd 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -3,12 +3,40 @@ from typing import Any import zarr.api.asynchronous as async_api -from zarr.array import Array, AsyncArray -from zarr.buffer import NDArrayLike -from zarr.common import JSON, AccessModeLiteral, ChunkCoords, ZarrFormat -from zarr.group import Group +from zarr.core.array import Array, AsyncArray +from zarr.core.buffer import NDArrayLike +from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, ZarrFormat +from zarr.core.group import Group +from zarr.core.sync import sync from zarr.store import StoreLike -from zarr.sync import sync + +__all__ = [ + "consolidate_metadata", + "copy", + "copy_all", + "copy_store", + "load", + "open", + "open_consolidated", + "save", + "save_array", + "save_group", + "tree", + "array", + "group", + "open_group", + "create", + "empty", + "empty_like", + "full", + "full_like", + "ones", + "ones_like", + "open_array", + "open_like", + "zeros", + "zeros_like", +] def consolidate_metadata(*args: Any, **kwargs: Any) -> Group: diff --git a/src/zarr/codecs/_v2.py b/src/zarr/codecs/_v2.py index 60854bee34..3dc5c8e4a7 100644 --- a/src/zarr/codecs/_v2.py +++ b/src/zarr/codecs/_v2.py @@ -6,9 +6,9 @@ from numcodecs.compat import ensure_bytes, ensure_ndarray from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec -from zarr.array_spec import ArraySpec -from zarr.buffer import Buffer, NDBuffer, default_buffer_prototype -from zarr.common import JSON, to_thread +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import Buffer, NDBuffer, default_buffer_prototype +from zarr.core.common import JSON, to_thread from zarr.registry import get_ndbuffer_class diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index d03ecbcbf2..8e01524992 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -9,9 +9,9 @@ from numcodecs.blosc import Blosc from zarr.abc.codec import BytesBytesCodec -from zarr.array_spec import ArraySpec -from zarr.buffer import Buffer, as_numpy_array_wrapper -from zarr.common import JSON, parse_enum, parse_named_configuration, to_thread +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import Buffer, as_numpy_array_wrapper +from zarr.core.common import JSON, parse_enum, parse_named_configuration, to_thread from zarr.registry import register_codec if TYPE_CHECKING: diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 1fe9d900eb..c2ef0ee4f6 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -8,9 +8,9 @@ import numpy as np from zarr.abc.codec import ArrayBytesCodec -from zarr.array_spec import ArraySpec -from zarr.buffer import Buffer, NDArrayLike, NDBuffer -from zarr.common import JSON, parse_enum, parse_named_configuration +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer +from zarr.core.common import JSON, parse_enum, parse_named_configuration from zarr.registry import register_codec if TYPE_CHECKING: diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index 5c94558b00..d40b95ef0c 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -7,9 +7,9 @@ from crc32c import crc32c from zarr.abc.codec import BytesBytesCodec -from zarr.array_spec import ArraySpec -from zarr.buffer import Buffer -from zarr.common import JSON, parse_named_configuration +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import Buffer +from zarr.core.common import JSON, parse_named_configuration from zarr.registry import register_codec if TYPE_CHECKING: diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py index 915ae79832..bab81a104b 100644 --- a/src/zarr/codecs/gzip.py +++ b/src/zarr/codecs/gzip.py @@ -6,9 +6,9 @@ from numcodecs.gzip import GZip from zarr.abc.codec import BytesBytesCodec -from zarr.array_spec import ArraySpec -from zarr.buffer import Buffer, as_numpy_array_wrapper -from zarr.common import JSON, parse_named_configuration, to_thread +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import Buffer, as_numpy_array_wrapper +from zarr.core.common import JSON, parse_named_configuration, to_thread from zarr.registry import register_codec if TYPE_CHECKING: diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/codecs/pipeline.py index 1582cd6adb..9c615d3d2c 100644 --- a/src/zarr/codecs/pipeline.py +++ b/src/zarr/codecs/pipeline.py @@ -18,17 +18,17 @@ CodecPipeline, ) from zarr.abc.store import ByteGetter, ByteSetter -from zarr.buffer import Buffer, BufferPrototype, NDBuffer -from zarr.chunk_grids import ChunkGrid -from zarr.common import JSON, ChunkCoords, concurrent_map, parse_named_configuration -from zarr.config import config -from zarr.indexing import SelectorTuple, is_scalar, is_total_slice +from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer +from zarr.core.chunk_grids import ChunkGrid +from zarr.core.common import JSON, ChunkCoords, concurrent_map, parse_named_configuration +from zarr.core.config import config +from zarr.core.indexing import SelectorTuple, is_scalar, is_total_slice from zarr.registry import get_codec_class, register_pipeline if TYPE_CHECKING: from typing_extensions import Self - from zarr.array_spec import ArraySpec + from zarr.core.array_spec import ArraySpec T = TypeVar("T") U = TypeVar("U") diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index be236beff1..fa706f57c7 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -18,18 +18,18 @@ CodecPipeline, ) from zarr.abc.store import ByteGetter, ByteSetter -from zarr.array_spec import ArraySpec -from zarr.buffer import ( +from zarr.codecs.bytes import BytesCodec +from zarr.codecs.crc32c_ import Crc32cCodec +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import ( Buffer, BufferPrototype, NDBuffer, default_buffer_prototype, numpy_buffer_prototype, ) -from zarr.chunk_grids import ChunkGrid, RegularChunkGrid -from zarr.codecs.bytes import BytesCodec -from zarr.codecs.crc32c_ import Crc32cCodec -from zarr.common import ( +from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid +from zarr.core.common import ( ChunkCoords, ChunkCoordsLike, parse_enum, @@ -37,8 +37,14 @@ parse_shapelike, product, ) -from zarr.indexing import BasicIndexer, SelectorTuple, c_order_iter, get_indexer, morton_order_iter -from zarr.metadata import parse_codecs +from zarr.core.indexing import ( + BasicIndexer, + SelectorTuple, + c_order_iter, + get_indexer, + morton_order_iter, +) +from zarr.core.metadata import parse_codecs from zarr.registry import get_ndbuffer_class, get_pipeline_class, register_codec if TYPE_CHECKING: @@ -46,7 +52,7 @@ from typing_extensions import Self - from zarr.common import JSON + from zarr.core.common import JSON MAX_UINT_64 = 2**64 - 1 ShardMapping = Mapping[ChunkCoords, Buffer] diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index 3f9ae61676..ada45b1b53 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -7,10 +7,10 @@ import numpy as np from zarr.abc.codec import ArrayArrayCodec -from zarr.array_spec import ArraySpec -from zarr.buffer import NDBuffer -from zarr.chunk_grids import ChunkGrid -from zarr.common import JSON, ChunkCoordsLike, parse_named_configuration +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import NDBuffer +from zarr.core.chunk_grids import ChunkGrid +from zarr.core.common import JSON, ChunkCoordsLike, parse_named_configuration from zarr.registry import register_codec if TYPE_CHECKING: diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index 3b3d3f33dd..92a2665425 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -8,9 +8,9 @@ from numcodecs.zstd import Zstd from zarr.abc.codec import BytesBytesCodec -from zarr.array_spec import ArraySpec -from zarr.buffer import Buffer, as_numpy_array_wrapper -from zarr.common import JSON, parse_named_configuration, to_thread +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import Buffer, as_numpy_array_wrapper +from zarr.core.common import JSON, parse_named_configuration, to_thread from zarr.registry import register_codec if TYPE_CHECKING: diff --git a/src/zarr/convenience.py b/src/zarr/convenience.py index be0a6b2813..f0ff602d07 100644 --- a/src/zarr/convenience.py +++ b/src/zarr/convenience.py @@ -14,22 +14,22 @@ tree, ) -warnings.warn( - "zarr.convenience is deprecated, use zarr.api.synchronous", - DeprecationWarning, - stacklevel=2, -) - __all__ = [ "open", - "save_array", - "save_group", "save", "load", - "tree", - "copy_store", + "save_array", + "save_group", "copy", "copy_all", + "copy_store", + "tree", "consolidate_metadata", "open_consolidated", ] + +warnings.warn( + "zarr.convenience is deprecated, use zarr.api.synchronous", + DeprecationWarning, + stacklevel=2, +) diff --git a/src/zarr/core/__init__.py b/src/zarr/core/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/zarr/array.py b/src/zarr/core/array.py similarity index 98% rename from src/zarr/array.py rename to src/zarr/core/array.py index 7aaf9e15ba..3738b0859b 100644 --- a/src/zarr/array.py +++ b/src/zarr/core/array.py @@ -1,14 +1,6 @@ from __future__ import annotations import json - -# Notes on what I've changed here: -# 1. Split Array into AsyncArray and Array -# 3. Added .size and .attrs methods -# 4. Temporarily disabled the creation of ArrayV2 -# 5. Added from_dict to AsyncArray -# Questions to consider: -# 1. Was splitting the array into two classes really necessary? from asyncio import gather from collections.abc import Iterable from dataclasses import dataclass, field, replace @@ -19,13 +11,17 @@ from zarr.abc.codec import Codec, CodecPipeline from zarr.abc.store import set_or_delete -from zarr.attributes import Attributes -from zarr.buffer import BufferPrototype, NDArrayLike, NDBuffer, default_buffer_prototype -from zarr.chunk_grids import RegularChunkGrid, _guess_chunks -from zarr.chunk_key_encodings import ChunkKeyEncoding, DefaultChunkKeyEncoding, V2ChunkKeyEncoding from zarr.codecs import BytesCodec from zarr.codecs._v2 import V2Compressor, V2Filters -from zarr.common import ( +from zarr.core.attributes import Attributes +from zarr.core.buffer import BufferPrototype, NDArrayLike, NDBuffer, default_buffer_prototype +from zarr.core.chunk_grids import RegularChunkGrid, _guess_chunks +from zarr.core.chunk_key_encodings import ( + ChunkKeyEncoding, + DefaultChunkKeyEncoding, + V2ChunkKeyEncoding, +) +from zarr.core.common import ( JSON, ZARR_JSON, ZARRAY_JSON, @@ -35,8 +31,8 @@ concurrent_map, product, ) -from zarr.config import config, parse_indexing_order -from zarr.indexing import ( +from zarr.core.config import config, parse_indexing_order +from zarr.core.indexing import ( BasicIndexer, BasicSelection, BlockIndex, @@ -59,13 +55,13 @@ is_scalar, pop_fields, ) -from zarr.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata +from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata +from zarr.core.sync import sync from zarr.registry import get_pipeline_class from zarr.store import StoreLike, StorePath, make_store_path -from zarr.store.core import ( +from zarr.store.common import ( ensure_no_existing_node, ) -from zarr.sync import sync def parse_array_metadata(data: Any) -> ArrayV2Metadata | ArrayV3Metadata: diff --git a/src/zarr/array_spec.py b/src/zarr/core/array_spec.py similarity index 80% rename from src/zarr/array_spec.py rename to src/zarr/core/array_spec.py index d5717944b4..d2f46057e4 100644 --- a/src/zarr/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -1,12 +1,15 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal import numpy as np -from zarr.buffer import BufferPrototype -from zarr.common import ChunkCoords, parse_dtype, parse_fill_value, parse_order, parse_shapelike +from zarr.core.common import parse_dtype, parse_fill_value, parse_order, parse_shapelike + +if TYPE_CHECKING: + from zarr.core.buffer import BufferPrototype + from zarr.core.common import ChunkCoords @dataclass(frozen=True) diff --git a/src/zarr/attributes.py b/src/zarr/core/attributes.py similarity index 82% rename from src/zarr/attributes.py rename to src/zarr/core/attributes.py index 079ae38a33..09677f7bdc 100644 --- a/src/zarr/attributes.py +++ b/src/zarr/core/attributes.py @@ -1,13 +1,15 @@ from __future__ import annotations -from collections.abc import Iterator, MutableMapping +from collections.abc import MutableMapping from typing import TYPE_CHECKING -from zarr.common import JSON +from zarr.core.common import JSON if TYPE_CHECKING: - from zarr.array import Array - from zarr.group import Group + from collections.abc import Iterator + + from zarr.core.array import Array + from zarr.core.group import Group class Attributes(MutableMapping[str, JSON]): diff --git a/src/zarr/buffer.py b/src/zarr/core/buffer.py similarity index 99% rename from src/zarr/buffer.py rename to src/zarr/core/buffer.py index 9c75ba1410..50252590dd 100644 --- a/src/zarr/buffer.py +++ b/src/zarr/core/buffer.py @@ -15,7 +15,7 @@ import numpy as np import numpy.typing as npt -from zarr.common import ChunkCoords +from zarr.core.common import ChunkCoords from zarr.registry import ( get_buffer_class, get_ndbuffer_class, @@ -27,7 +27,7 @@ from typing_extensions import Self from zarr.codecs.bytes import Endian - from zarr.common import BytesLike + from zarr.core.common import BytesLike @runtime_checkable diff --git a/src/zarr/chunk_grids.py b/src/zarr/core/chunk_grids.py similarity index 98% rename from src/zarr/chunk_grids.py rename to src/zarr/core/chunk_grids.py index b494a3e5e8..f912872251 100644 --- a/src/zarr/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -12,14 +12,14 @@ import numpy as np from zarr.abc.metadata import Metadata -from zarr.common import ( +from zarr.core.common import ( JSON, ChunkCoords, ChunkCoordsLike, parse_named_configuration, parse_shapelike, ) -from zarr.indexing import ceildiv +from zarr.core.indexing import ceildiv if TYPE_CHECKING: from typing_extensions import Self diff --git a/src/zarr/chunk_key_encodings.py b/src/zarr/core/chunk_key_encodings.py similarity index 98% rename from src/zarr/chunk_key_encodings.py rename to src/zarr/core/chunk_key_encodings.py index 30db8ff5a8..45174bf4af 100644 --- a/src/zarr/chunk_key_encodings.py +++ b/src/zarr/core/chunk_key_encodings.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Literal, cast from zarr.abc.metadata import Metadata -from zarr.common import ( +from zarr.core.common import ( JSON, ChunkCoords, parse_named_configuration, diff --git a/src/zarr/common.py b/src/zarr/core/common.py similarity index 100% rename from src/zarr/common.py rename to src/zarr/core/common.py diff --git a/src/zarr/config.py b/src/zarr/core/config.py similarity index 96% rename from src/zarr/config.py rename to src/zarr/core/config.py index ec78747a6b..67f5c74347 100644 --- a/src/zarr/config.py +++ b/src/zarr/core/config.py @@ -60,8 +60,8 @@ def reset(self) -> None: "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", "transpose": "zarr.codecs.transpose.TransposeCodec", }, - "buffer": "zarr.buffer.Buffer", - "ndbuffer": "zarr.buffer.NDBuffer", + "buffer": "zarr.core.buffer.Buffer", + "ndbuffer": "zarr.core.buffer.NDBuffer", } ], ) diff --git a/src/zarr/group.py b/src/zarr/core/group.py similarity index 98% rename from src/zarr/group.py rename to src/zarr/core/group.py index a02fa6196a..86d27e3a97 100644 --- a/src/zarr/group.py +++ b/src/zarr/core/group.py @@ -13,11 +13,11 @@ from zarr.abc.codec import Codec from zarr.abc.metadata import Metadata from zarr.abc.store import set_or_delete -from zarr.array import Array, AsyncArray -from zarr.attributes import Attributes -from zarr.buffer import Buffer, BufferPrototype, default_buffer_prototype -from zarr.chunk_key_encodings import ChunkKeyEncoding -from zarr.common import ( +from zarr.core.array import Array, AsyncArray +from zarr.core.attributes import Attributes +from zarr.core.buffer import default_buffer_prototype +from zarr.core.chunk_key_encodings import ChunkKeyEncoding +from zarr.core.common import ( JSON, ZARR_JSON, ZARRAY_JSON, @@ -26,15 +26,17 @@ ChunkCoords, ZarrFormat, ) -from zarr.config import config +from zarr.core.config import config +from zarr.core.sync import SyncMixin, sync from zarr.store import StoreLike, StorePath, make_store_path -from zarr.store.core import ensure_no_existing_node -from zarr.sync import SyncMixin, sync +from zarr.store.common import ensure_no_existing_node if TYPE_CHECKING: from collections.abc import AsyncGenerator, Iterable from typing import Any + from zarr.core.buffer import Buffer, BufferPrototype + logger = logging.getLogger("zarr.group") diff --git a/src/zarr/indexing.py b/src/zarr/core/indexing.py similarity index 99% rename from src/zarr/indexing.py rename to src/zarr/core/indexing.py index 6987f69c11..153483330c 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/core/indexing.py @@ -23,12 +23,13 @@ import numpy as np import numpy.typing as npt -from zarr.buffer import NDArrayLike -from zarr.common import ChunkCoords, product +from zarr.core.buffer import NDArrayLike +from zarr.core.common import product if TYPE_CHECKING: - from zarr.array import Array - from zarr.chunk_grids import ChunkGrid + from zarr.core.array import Array + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.common import ChunkCoords IntSequence = list[int] | npt.NDArray[np.intp] ArrayOfIntOrBool = npt.NDArray[np.intp] | npt.NDArray[np.bool_] @@ -197,7 +198,7 @@ def is_pure_orthogonal_indexing(selection: Selection, ndim: int) -> TypeGuard[Or def get_chunk_shape(chunk_grid: ChunkGrid) -> ChunkCoords: - from zarr.chunk_grids import RegularChunkGrid + from zarr.core.chunk_grids import RegularChunkGrid assert isinstance( chunk_grid, RegularChunkGrid diff --git a/src/zarr/metadata.py b/src/zarr/core/metadata.py similarity index 98% rename from src/zarr/metadata.py rename to src/zarr/core/metadata.py index e801a6f966..d541e43205 100644 --- a/src/zarr/metadata.py +++ b/src/zarr/core/metadata.py @@ -12,10 +12,9 @@ from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec, CodecPipeline from zarr.abc.metadata import Metadata -from zarr.buffer import Buffer, BufferPrototype, default_buffer_prototype -from zarr.chunk_grids import ChunkGrid, RegularChunkGrid -from zarr.chunk_key_encodings import ChunkKeyEncoding, parse_separator -from zarr.config import config +from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype +from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid +from zarr.core.chunk_key_encodings import ChunkKeyEncoding, parse_separator from zarr.registry import get_codec_class, get_pipeline_class if TYPE_CHECKING: @@ -23,8 +22,8 @@ import numcodecs.abc -from zarr.array_spec import ArraySpec -from zarr.common import ( +from zarr.core.array_spec import ArraySpec +from zarr.core.common import ( JSON, ZARR_JSON, ZARRAY_JSON, @@ -35,7 +34,7 @@ parse_named_configuration, parse_shapelike, ) -from zarr.config import parse_indexing_order +from zarr.core.config import config, parse_indexing_order # For type checking _bool = bool diff --git a/src/zarr/sync.py b/src/zarr/core/sync.py similarity index 99% rename from src/zarr/sync.py rename to src/zarr/core/sync.py index 8af14f602e..ff7f9a43af 100644 --- a/src/zarr/sync.py +++ b/src/zarr/core/sync.py @@ -1,18 +1,17 @@ from __future__ import annotations -from typing import TYPE_CHECKING, TypeVar - -if TYPE_CHECKING: - from collections.abc import AsyncIterator, Coroutine - from typing import Any - import asyncio import threading from concurrent.futures import wait +from typing import TYPE_CHECKING, TypeVar from typing_extensions import ParamSpec -from zarr.config import config +from zarr.core.config import config + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Coroutine + from typing import Any P = ParamSpec("P") T = TypeVar("T") diff --git a/src/zarr/creation.py b/src/zarr/creation.py index df3f764610..aa4b5a8088 100644 --- a/src/zarr/creation.py +++ b/src/zarr/creation.py @@ -15,12 +15,6 @@ zeros_like, ) -warnings.warn( - "zarr.creation is deprecated, use zarr.api.synchronous", - DeprecationWarning, - stacklevel=2, -) - __all__ = [ "create", "empty", @@ -35,3 +29,9 @@ "full_like", "open_like", ] + +warnings.warn( + "zarr.creation is deprecated, use zarr.api.synchronous", + DeprecationWarning, + stacklevel=2, +) diff --git a/src/zarr/errors.py b/src/zarr/errors.py index 140229b2eb..f6321e474b 100644 --- a/src/zarr/errors.py +++ b/src/zarr/errors.py @@ -23,3 +23,10 @@ class ContainsArrayAndGroupError(_BaseZarrError): "Only one of these files may be present in a given directory / prefix. " "Remove the .zarray file, or the .zgroup file, or both." ) + + +__all__ = [ + "ContainsGroupError", + "ContainsArrayError", + "ContainsArrayAndGroupError", +] diff --git a/src/zarr/registry.py b/src/zarr/registry.py index ac373f401d..cde3b7d848 100644 --- a/src/zarr/registry.py +++ b/src/zarr/registry.py @@ -2,16 +2,28 @@ import warnings from collections import defaultdict +from importlib.metadata import entry_points as get_entry_points from typing import TYPE_CHECKING, Any, Generic, TypeVar -if TYPE_CHECKING: - from zarr.abc.codec import Codec, CodecPipeline - from zarr.buffer import Buffer, NDBuffer +from zarr.core.config import BadConfigError, config -from importlib.metadata import EntryPoint -from importlib.metadata import entry_points as get_entry_points +if TYPE_CHECKING: + from importlib.metadata import EntryPoint -from zarr.config import BadConfigError, config + from zarr.abc.codec import Codec, CodecPipeline + from zarr.core.buffer import Buffer, NDBuffer + +__all__ = [ + "Registry", + "register_codec", + "register_pipeline", + "register_buffer", + "register_ndbuffer", + "get_codec_class", + "get_pipeline_class", + "get_buffer_class", + "get_ndbuffer_class", +] T = TypeVar("T") diff --git a/src/zarr/store/__init__.py b/src/zarr/store/__init__.py index fbdcdb9255..3a9e8201e8 100644 --- a/src/zarr/store/__init__.py +++ b/src/zarr/store/__init__.py @@ -1,4 +1,4 @@ -from zarr.store.core import StoreLike, StorePath, make_store_path +from zarr.store.common import StoreLike, StorePath, make_store_path from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore from zarr.store.remote import RemoteStore diff --git a/src/zarr/store/utils.py b/src/zarr/store/_utils.py similarity index 94% rename from src/zarr/store/utils.py rename to src/zarr/store/_utils.py index 17c9234221..04a06351c5 100644 --- a/src/zarr/store/utils.py +++ b/src/zarr/store/_utils.py @@ -1,4 +1,4 @@ -from zarr.buffer import Buffer +from zarr.core.buffer import Buffer def _normalize_interval_index( diff --git a/src/zarr/store/core.py b/src/zarr/store/common.py similarity index 96% rename from src/zarr/store/core.py rename to src/zarr/store/common.py index 4d31118a54..6a88de7760 100644 --- a/src/zarr/store/core.py +++ b/src/zarr/store/common.py @@ -2,15 +2,19 @@ import json from pathlib import Path -from typing import Any, Literal +from typing import TYPE_CHECKING, Any, Literal from zarr.abc.store import AccessMode, Store -from zarr.buffer import Buffer, BufferPrototype, default_buffer_prototype -from zarr.common import ZARR_JSON, ZARRAY_JSON, ZGROUP_JSON, AccessModeLiteral, ZarrFormat +from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.core.common import ZARR_JSON, ZARRAY_JSON, ZGROUP_JSON, ZarrFormat from zarr.errors import ContainsArrayAndGroupError, ContainsArrayError, ContainsGroupError from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore +if TYPE_CHECKING: + from zarr.core.buffer import BufferPrototype + from zarr.core.common import AccessModeLiteral + def _dereference_path(root: str, path: str) -> str: assert isinstance(root, str) diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index 3cc42c6c0e..6510305844 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -5,10 +5,15 @@ import shutil from collections.abc import AsyncGenerator from pathlib import Path +from typing import TYPE_CHECKING from zarr.abc.store import Store -from zarr.buffer import Buffer, BufferPrototype -from zarr.common import AccessModeLiteral, concurrent_map, to_thread +from zarr.core.buffer import Buffer +from zarr.core.common import concurrent_map, to_thread + +if TYPE_CHECKING: + from zarr.core.buffer import BufferPrototype + from zarr.core.common import AccessModeLiteral def _get( diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index dd3e52e703..999d750755 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -1,11 +1,16 @@ from __future__ import annotations from collections.abc import AsyncGenerator, MutableMapping +from typing import TYPE_CHECKING from zarr.abc.store import Store -from zarr.buffer import Buffer, BufferPrototype -from zarr.common import AccessModeLiteral, concurrent_map -from zarr.store.utils import _normalize_interval_index +from zarr.core.buffer import Buffer +from zarr.core.common import concurrent_map +from zarr.store._utils import _normalize_interval_index + +if TYPE_CHECKING: + from zarr.core.buffer import BufferPrototype + from zarr.core.common import AccessModeLiteral # TODO: this store could easily be extended to wrap any MutableMapping store from v2 diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index 84f01c7852..f5ea694b0a 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -6,16 +6,15 @@ import fsspec from zarr.abc.store import Store -from zarr.buffer import BufferPrototype -from zarr.common import AccessModeLiteral -from zarr.store.core import _dereference_path +from zarr.core.buffer import Buffer +from zarr.store.common import _dereference_path if TYPE_CHECKING: from fsspec.asyn import AsyncFileSystem from upath import UPath - from zarr.buffer import Buffer, BufferPrototype - from zarr.common import BytesLike + from zarr.core.buffer import BufferPrototype + from zarr.core.common import AccessModeLiteral, BytesLike class RemoteStore(Store): diff --git a/src/zarr/strategies.py b/src/zarr/strategies.py index 91a8542ce9..bfd83e1e51 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/strategies.py @@ -5,9 +5,8 @@ import numpy as np from hypothesis import given, settings # noqa -from .array import Array -from .group import Group -from .store import MemoryStore, StoreLike +from zarr import Array, Group +from zarr.store import MemoryStore, StoreLike # Copied from Xarray _attr_keys = st.text(st.characters(), min_size=1) diff --git a/src/zarr/testing/__init__.py b/src/zarr/testing/__init__.py index 35b91f9167..0b4d8cf417 100644 --- a/src/zarr/testing/__init__.py +++ b/src/zarr/testing/__init__.py @@ -6,4 +6,8 @@ else: warnings.warn("pytest not installed, skipping test suite", stacklevel=2) -__all__ = ["StoreTests"] +from zarr.testing.utils import assert_bytes_equal + +# TODO: import public buffer tests? + +__all__ = ["StoreTests", "assert_bytes_equal"] diff --git a/src/zarr/testing/buffer.py b/src/zarr/testing/buffer.py index d2da1c5a6e..ee170a5dd3 100644 --- a/src/zarr/testing/buffer.py +++ b/src/zarr/testing/buffer.py @@ -7,13 +7,20 @@ import numpy as np import numpy.typing as npt -from zarr.buffer import Buffer, BufferPrototype, NDBuffer +from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer from zarr.store import MemoryStore if TYPE_CHECKING: from typing_extensions import Self +__all__ = [ + "TestBuffer", + "NDBufferUsingTestNDArrayLike", + "StoreExpectingTestBuffer", +] + + class TestNDArrayLike(np.ndarray): """An example of a ndarray-like class""" diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 4fdf497a68..ebef4824f7 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -3,10 +3,13 @@ import pytest from zarr.abc.store import AccessMode, Store -from zarr.buffer import Buffer, default_buffer_prototype -from zarr.store.utils import _normalize_interval_index +from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.store._utils import _normalize_interval_index from zarr.testing.utils import assert_bytes_equal +__all__ = ["StoreTests"] + + S = TypeVar("S", bound=Store) diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 67c6c72de7..3a70f96d44 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,7 +1,9 @@ from __future__ import annotations -from zarr.buffer import Buffer -from zarr.common import BytesLike +from zarr.core.buffer import Buffer +from zarr.core.common import BytesLike + +__all__ = ["assert_bytes_equal"] def assert_bytes_equal(b1: Buffer | BytesLike | None, b2: Buffer | BytesLike | None) -> None: diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index 0a672d1f2e..d8af484ee6 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -6,10 +6,9 @@ from _pytest.compat import LEGACY_PATH -from zarr import config +from zarr import AsyncGroup, config from zarr.abc.store import Store -from zarr.common import ChunkCoords, MemoryOrder, ZarrFormat -from zarr.group import AsyncGroup +from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat if TYPE_CHECKING: from typing import Any, Literal diff --git a/tests/v3/package_with_entrypoint/__init__.py b/tests/v3/package_with_entrypoint/__init__.py index 4d626808d8..0e84cde0d5 100644 --- a/tests/v3/package_with_entrypoint/__init__.py +++ b/tests/v3/package_with_entrypoint/__init__.py @@ -3,10 +3,10 @@ from numpy import ndarray from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline -from zarr.array_spec import ArraySpec -from zarr.buffer import Buffer, NDBuffer from zarr.codecs import BytesCodec -from zarr.common import BytesLike +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import Buffer, NDBuffer +from zarr.core.common import BytesLike class TestEntrypointCodec(ArrayBytesCodec): diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index 9fd135ad5c..fb726757dc 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -3,12 +3,11 @@ import numpy as np import pytest -from zarr.array import Array -from zarr.common import ZarrFormat +from zarr import Array, Group +from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError -from zarr.group import Group from zarr.store import LocalStore, MemoryStore -from zarr.store.core import StorePath +from zarr.store.common import StorePath @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index d53e98d42d..298a7f7eed 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -3,15 +3,15 @@ import numpy as np import pytest -from zarr.array import AsyncArray -from zarr.buffer import ArrayLike, BufferPrototype, NDArrayLike, numpy_buffer_prototype +from zarr import AsyncArray from zarr.codecs.blosc import BloscCodec from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec -from zarr.store.core import StorePath +from zarr.core.buffer import ArrayLike, BufferPrototype, NDArrayLike, numpy_buffer_prototype +from zarr.store.common import StorePath from zarr.testing.buffer import ( NDBufferUsingTestNDArrayLike, StoreExpectingTestBuffer, diff --git a/tests/v3/test_chunk_grids.py b/tests/v3/test_chunk_grids.py index 3cc6b64e57..e1b4df10a7 100644 --- a/tests/v3/test_chunk_grids.py +++ b/tests/v3/test_chunk_grids.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from zarr.chunk_grids import _guess_chunks +from zarr.core.chunk_grids import _guess_chunks @pytest.mark.parametrize( diff --git a/tests/v3/test_codecs/test_blosc.py b/tests/v3/test_codecs/test_blosc.py index 33ca9eba77..5de4c9fa99 100644 --- a/tests/v3/test_codecs/test_blosc.py +++ b/tests/v3/test_codecs/test_blosc.py @@ -3,11 +3,11 @@ import numpy as np import pytest +from zarr import AsyncArray from zarr.abc.store import Store -from zarr.array import AsyncArray -from zarr.buffer import default_buffer_prototype from zarr.codecs import BloscCodec, BytesCodec, ShardingCodec -from zarr.store.core import StorePath +from zarr.core.buffer import default_buffer_prototype +from zarr.store.common import StorePath @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/v3/test_codecs/test_codecs.py index 2a1b99c42f..7e717e5b50 100644 --- a/tests/v3/test_codecs/test_codecs.py +++ b/tests/v3/test_codecs/test_codecs.py @@ -7,19 +7,18 @@ import pytest import zarr.v2 +from zarr import Array, AsyncArray, config from zarr.abc.codec import Codec from zarr.abc.store import Store -from zarr.array import Array, AsyncArray -from zarr.buffer import default_buffer_prototype from zarr.codecs import ( BytesCodec, GzipCodec, ShardingCodec, TransposeCodec, ) -from zarr.common import MemoryOrder -from zarr.config import config -from zarr.indexing import Selection, morton_order_iter +from zarr.core.buffer import default_buffer_prototype +from zarr.core.common import MemoryOrder +from zarr.core.indexing import Selection, morton_order_iter from zarr.store import StorePath from zarr.testing.utils import assert_bytes_equal diff --git a/tests/v3/test_codecs/test_endian.py b/tests/v3/test_codecs/test_endian.py index 6f3e1c9482..f97d95d9b7 100644 --- a/tests/v3/test_codecs/test_endian.py +++ b/tests/v3/test_codecs/test_endian.py @@ -4,11 +4,11 @@ import pytest import zarr.v2 +from zarr import AsyncArray from zarr.abc.store import Store -from zarr.array import AsyncArray -from zarr.buffer import default_buffer_prototype from zarr.codecs import BytesCodec -from zarr.store.core import StorePath +from zarr.core.buffer import default_buffer_prototype +from zarr.store.common import StorePath from zarr.testing.utils import assert_bytes_equal from .test_codecs import _AsyncArrayProxy diff --git a/tests/v3/test_codecs/test_gzip.py b/tests/v3/test_codecs/test_gzip.py index f982bacb87..6495f8236c 100644 --- a/tests/v3/test_codecs/test_gzip.py +++ b/tests/v3/test_codecs/test_gzip.py @@ -1,10 +1,10 @@ import numpy as np import pytest +from zarr import Array from zarr.abc.store import Store -from zarr.array import Array from zarr.codecs import BytesCodec, GzipCodec -from zarr.store.core import StorePath +from zarr.store.common import StorePath @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) diff --git a/tests/v3/test_codecs/test_sharding.py b/tests/v3/test_codecs/test_sharding.py index 27667ca9dd..e5f66224e9 100644 --- a/tests/v3/test_codecs/test_sharding.py +++ b/tests/v3/test_codecs/test_sharding.py @@ -3,9 +3,8 @@ import numpy as np import pytest +from zarr import Array, AsyncArray from zarr.abc.store import Store -from zarr.array import Array, AsyncArray -from zarr.buffer import default_buffer_prototype from zarr.codecs import ( BloscCodec, BytesCodec, @@ -13,7 +12,8 @@ ShardingCodecIndexLocation, TransposeCodec, ) -from zarr.store.core import StorePath +from zarr.core.buffer import default_buffer_prototype +from zarr.store.common import StorePath from ..conftest import ArrayRequest from .test_codecs import _AsyncArrayProxy, order_from_dim diff --git a/tests/v3/test_codecs/test_transpose.py b/tests/v3/test_codecs/test_transpose.py index bea7435122..b9aa6cc4b6 100644 --- a/tests/v3/test_codecs/test_transpose.py +++ b/tests/v3/test_codecs/test_transpose.py @@ -2,14 +2,13 @@ import pytest import zarr.v2 +from zarr import Array, AsyncArray, config from zarr.abc.codec import Codec from zarr.abc.store import Store -from zarr.array import Array, AsyncArray -from zarr.buffer import default_buffer_prototype from zarr.codecs import BytesCodec, ShardingCodec, TransposeCodec -from zarr.common import MemoryOrder -from zarr.config import config -from zarr.store.core import StorePath +from zarr.core.buffer import default_buffer_prototype +from zarr.core.common import MemoryOrder +from zarr.store.common import StorePath from .test_codecs import _AsyncArrayProxy diff --git a/tests/v3/test_codecs/test_zstd.py b/tests/v3/test_codecs/test_zstd.py index 1e1b1e02c9..0726e5944c 100644 --- a/tests/v3/test_codecs/test_zstd.py +++ b/tests/v3/test_codecs/test_zstd.py @@ -1,10 +1,10 @@ import numpy as np import pytest +from zarr import Array from zarr.abc.store import Store -from zarr.array import Array from zarr.codecs import BytesCodec, ZstdCodec -from zarr.store.core import StorePath +from zarr.store.common import StorePath @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) diff --git a/tests/v3/test_common.py b/tests/v3/test_common.py index bb59789843..f919b4d5f7 100644 --- a/tests/v3/test_common.py +++ b/tests/v3/test_common.py @@ -9,8 +9,8 @@ import numpy as np import pytest -from zarr.common import parse_name, parse_shapelike, product -from zarr.config import parse_indexing_order +from zarr.core.common import parse_name, parse_shapelike, product +from zarr.core.config import parse_indexing_order @pytest.mark.parametrize("data", [(0, 0, 0, 0), (1, 3, 4, 5, 6), (2, 4)]) diff --git a/tests/v3/test_config.py b/tests/v3/test_config.py index 8e7b868520..881833797c 100644 --- a/tests/v3/test_config.py +++ b/tests/v3/test_config.py @@ -11,11 +11,11 @@ from zarr import Array, zeros from zarr.abc.codec import CodecInput, CodecOutput, CodecPipeline from zarr.abc.store import ByteSetter -from zarr.array_spec import ArraySpec -from zarr.buffer import NDBuffer from zarr.codecs import BatchedCodecPipeline, BloscCodec, BytesCodec, Crc32cCodec, ShardingCodec -from zarr.config import BadConfigError, config -from zarr.indexing import SelectorTuple +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import NDBuffer +from zarr.core.config import BadConfigError, config +from zarr.core.indexing import SelectorTuple from zarr.registry import ( fully_qualified_name, get_buffer_class, @@ -46,8 +46,8 @@ def test_config_defaults_set() -> None: "path": "zarr.codecs.pipeline.BatchedCodecPipeline", "batch_size": 1, }, - "buffer": "zarr.buffer.Buffer", - "ndbuffer": "zarr.buffer.NDBuffer", + "buffer": "zarr.core.buffer.Buffer", + "ndbuffer": "zarr.core.buffer.NDBuffer", "codecs": { "blosc": "zarr.codecs.blosc.BloscCodec", "gzip": "zarr.codecs.gzip.GzipCodec", diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index daa5979b27..39921c26d8 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -6,14 +6,14 @@ import pytest from _pytest.compat import LEGACY_PATH -from zarr.array import Array, AsyncArray -from zarr.buffer import Buffer -from zarr.common import ZarrFormat +from zarr import Array, AsyncArray, AsyncGroup, Group +from zarr.core.buffer import Buffer +from zarr.core.common import ZarrFormat +from zarr.core.group import GroupMetadata +from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError -from zarr.group import AsyncGroup, Group, GroupMetadata from zarr.store import LocalStore, MemoryStore, StorePath -from zarr.store.core import make_store_path -from zarr.sync import sync +from zarr.store.common import make_store_path from .conftest import parse_store diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 12547765a9..7895151b5f 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -12,9 +12,9 @@ import zarr from zarr.abc.store import Store -from zarr.buffer import BufferPrototype, NDBuffer -from zarr.common import ChunkCoords -from zarr.indexing import ( +from zarr.core.buffer import BufferPrototype, NDBuffer +from zarr.core.common import ChunkCoords +from zarr.core.indexing import ( make_slice_selection, normalize_integer_selection, oindex, @@ -22,7 +22,7 @@ replace_ellipsis, ) from zarr.registry import get_ndbuffer_class -from zarr.store.core import StorePath +from zarr.store.common import StorePath from zarr.store.memory import MemoryStore diff --git a/tests/v3/test_metadata/test_v2.py b/tests/v3/test_metadata/test_v2.py index 2d6f4d0cf5..54f26d91ca 100644 --- a/tests/v3/test_metadata/test_v2.py +++ b/tests/v3/test_metadata/test_v2.py @@ -10,7 +10,7 @@ import pytest from zarr.codecs import GzipCodec -from zarr.metadata import ArrayV2Metadata, parse_zarr_format_v2 +from zarr.core.metadata import ArrayV2Metadata, parse_zarr_format_v2 def test_parse_zarr_format_valid() -> None: diff --git a/tests/v3/test_metadata/test_v3.py b/tests/v3/test_metadata/test_v3.py index 2b25c776e0..eedcdf6234 100644 --- a/tests/v3/test_metadata/test_v3.py +++ b/tests/v3/test_metadata/test_v3.py @@ -4,8 +4,8 @@ from typing import TYPE_CHECKING, Literal from zarr.abc.codec import Codec -from zarr.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding from zarr.codecs.bytes import BytesCodec +from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding if TYPE_CHECKING: from typing import Any @@ -15,9 +15,9 @@ import numpy as np import pytest -from zarr.metadata import ArrayV3Metadata, parse_dimension_names -from zarr.metadata import parse_fill_value_v3 as parse_fill_value -from zarr.metadata import parse_zarr_format_v3 as parse_zarr_format +from zarr.core.metadata import ArrayV3Metadata, parse_dimension_names +from zarr.core.metadata import parse_fill_value_v3 as parse_fill_value +from zarr.core.metadata import parse_zarr_format_v3 as parse_zarr_format bool_dtypes = ("bool",) diff --git a/tests/v3/test_store/test_core.py b/tests/v3/test_store/test_core.py index 1d277cf502..23821acfa6 100644 --- a/tests/v3/test_store/test_core.py +++ b/tests/v3/test_store/test_core.py @@ -2,7 +2,7 @@ import pytest -from zarr.store.core import make_store_path +from zarr.store.common import make_store_path from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore diff --git a/tests/v3/test_store/test_local.py b/tests/v3/test_store/test_local.py index 6b7f91b87d..8afa4fef3a 100644 --- a/tests/v3/test_store/test_local.py +++ b/tests/v3/test_store/test_local.py @@ -2,7 +2,7 @@ import pytest -from zarr.buffer import Buffer +from zarr.core.buffer import Buffer from zarr.store.local import LocalStore from zarr.testing.store import StoreTests diff --git a/tests/v3/test_store/test_memory.py b/tests/v3/test_store/test_memory.py index 5b8f1ef875..51ecf46709 100644 --- a/tests/v3/test_store/test_memory.py +++ b/tests/v3/test_store/test_memory.py @@ -2,7 +2,7 @@ import pytest -from zarr.buffer import Buffer +from zarr.core.buffer import Buffer from zarr.store.memory import MemoryStore from zarr.testing.store import StoreTests diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index be9fa5ef67..2eec9d29cf 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -4,9 +4,9 @@ import pytest from upath import UPath -from zarr.buffer import Buffer, default_buffer_prototype +from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.core.sync import sync from zarr.store import RemoteStore -from zarr.sync import sync from zarr.testing.store import StoreTests s3fs = pytest.importorskip("s3fs") diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 7e3b8dd111..a335f9b48c 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -4,7 +4,7 @@ import pytest -from zarr.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync +from zarr.core.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync @pytest.fixture(params=[True, False]) diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py index 7a7d728067..4f4dc5aed3 100644 --- a/tests/v3/test_v2.py +++ b/tests/v3/test_v2.py @@ -3,8 +3,8 @@ import numpy as np import pytest +from zarr import Array from zarr.abc.store import Store -from zarr.array import Array from zarr.store import MemoryStore, StorePath From 11fd8db8f06372ffc1f6f7e22315d43e4702ec60 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Wed, 14 Aug 2024 13:48:25 -0600 Subject: [PATCH 0111/1043] Fix orthogonal indexing with scalar. (#1947) Co-authored-by: David Stansby --- docs/release.rst | 11 +++++++++++ zarr/core.py | 7 ++++++- zarr/tests/test_core.py | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index 6c7ba5139b..c40ff5c0d3 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,17 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _release_2.18.3: + +2.18.3 +------ + +Maintenance +~~~~~~~~~~~ +* Fix a regression when using orthogonal indexing with a scalar. + By :user:`Deepak Cherian ` :issue:`1931` + + .. _release_2.18.2: Enhancements diff --git a/zarr/core.py b/zarr/core.py index 4f3080e46c..141190a0c8 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -2053,7 +2053,12 @@ def _process_chunk( if isinstance(cdata, UncompressedPartialReadBufferV3): cdata = cdata.read_full() chunk = ensure_ndarray_like(cdata).view(self._dtype) - chunk = chunk.reshape(self._chunks, order=self._order) + # dest.shape is not self._chunks when a dimensions is squeezed out + # For example, assume self._chunks = (5, 5, 1) + # and the selection is [:, :, 0] + # Then out_selection is (slice(5), slice(5)) + # See https://github.com/zarr-developers/zarr-python/issues/1931 + chunk = chunk.reshape(dest.shape, order=self._order) np.copyto(dest, chunk) return diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 01a78ecd68..4729dc01b6 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -3206,3 +3206,20 @@ def test_object_array_indexing(): elem = [1, 3] arr[1] = elem assert arr[1] == elem + + +@pytest.mark.parametrize("shape", ((1, 1, 1), (5, 5, 1), (1, 5, 5))) +def test_scalar_orthogonal_indexing(shape): + # regression test for https://github.com/zarr-developers/zarr-python/issues/1931 + store = zarr.MemoryStore({}) + data = np.random.randint(0, 255, shape) + arr = zarr.zeros( + shape=shape, chunks=shape[:-1] + (1,), compressor=None, store=store, dtype="u1" + ) + arr[:, :, :] = data + store.close() + + zf = zarr.open(store, "r") + assert_array_equal(zf[0, :, :], data[0, :, :]) + assert_array_equal(zf[:, 0, :], data[:, 0, :]) + assert_array_equal(zf[:, :, 0], data[:, :, 0]) From 9d0a050bf2949716fa9c4a15e38841bdcf816f1e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 15 Aug 2024 08:39:31 +0100 Subject: [PATCH 0112/1043] Re-enable dependabot on main branch (#2082) --- .github/dependabot.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 52fe24c7a7..a437a5c269 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,7 @@ --- version: 2 updates: + # Updates for v3 branch (the default branch) - package-ecosystem: "pip" directory: "/" schedule: @@ -17,3 +18,23 @@ updates: actions: patterns: - "*" + + # Same updates, but for main branch + - package-ecosystem: "pip" + directory: "/" + target-branch: "main" + schedule: + interval: "daily" + groups: + requirements: + patterns: + - "*" + - package-ecosystem: "github-actions" + directory: "/" + target-branch: "main" + schedule: + interval: "weekly" + groups: + actions: + patterns: + - "*" From 681f2866343963f8fff3123f2ac8e18e2bc4b3ad Mon Sep 17 00:00:00 2001 From: Emma Marshall <55526386+e-marshall@users.noreply.github.com> Date: Thu, 15 Aug 2024 15:30:10 -0600 Subject: [PATCH 0113/1043] Stateful store tests (#2070) * stateful store test * Deepak's edits * add data strategies and beginning of testgetpartial * a few more updates * troubleshooting get_partial * edits * Deepak edits * bundle changesc * add sts script, delete * comment out bundles for now * draft of listdir * edits * some changes * draft of stateful store tests * precommit * Update src/zarr/strategies.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Apply suggestions from code review Co-authored-by: Deepak Cherian * Apply suggestions from code review Co-authored-by: Deepak Cherian * moved strategies and edits to stateful store tests * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * Update tests/v3/test_store/test_stateful_store.py Co-authored-by: Deepak Cherian * fixed wrapper class and add rules for a few more methods * incorporate Deepak's edits * switch to bundles + a few more updates * remove some notes * Update src/zarr/testing/strategies.py Co-authored-by: Deepak Cherian * remove bundles * fix get signature, get invalid keys and some other changes * add docstrings * Deepak edits * fixing imports now that strategies in testing * revert docstrings * Update run-hypothesis command * Fix run-hypothesis command * Apply suggestions from code review Co-authored-by: Joe Hamman * changes from Joe's review * Apply suggestions from code review Co-authored-by: Joe Hamman * add review comments --------- Co-authored-by: Deepak Cherian Co-authored-by: Deepak Cherian Co-authored-by: Joe Hamman --- pyproject.toml | 2 +- src/zarr/{ => testing}/strategies.py | 17 +- tests/v3/test_properties.py | 2 +- tests/v3/test_store/test_stateful_store.py | 229 +++++++++++++++++++++ 4 files changed, 247 insertions(+), 3 deletions(-) rename src/zarr/{ => testing}/strategies.py (88%) create mode 100644 tests/v3/test_store/test_stateful_store.py diff --git a/pyproject.toml b/pyproject.toml index cf5c242406..197c4aca10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -139,7 +139,7 @@ run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" -run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py" +run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*" list-env = "pip list" [tool.hatch.envs.docs] diff --git a/src/zarr/strategies.py b/src/zarr/testing/strategies.py similarity index 88% rename from src/zarr/strategies.py rename to src/zarr/testing/strategies.py index bfd83e1e51..d2e41c6290 100644 --- a/src/zarr/strategies.py +++ b/src/zarr/testing/strategies.py @@ -4,8 +4,10 @@ import hypothesis.strategies as st import numpy as np from hypothesis import given, settings # noqa +from hypothesis.strategies import SearchStrategy -from zarr import Array, Group +from zarr.core.array import Array +from zarr.core.group import Group from zarr.store import MemoryStore, StoreLike # Copied from Xarray @@ -142,3 +144,16 @@ def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs): # type: ign ) ) ) + + +def key_ranges(keys: SearchStrategy = node_names) -> SearchStrategy[list]: + """fn to generate key_ranges strategy for get_partial_values() + returns list strategy w/ form: [(key, (range_start, range_step)), + (key, (range_start, range_step)),...] + """ + byte_ranges = st.tuples( + st.none() | st.integers(min_value=0), st.none() | st.integers(min_value=0) + ) + key_tuple = st.tuples(keys, byte_ranges) + key_range_st = st.lists(key_tuple, min_size=1, max_size=10) + return key_range_st diff --git a/tests/v3/test_properties.py b/tests/v3/test_properties.py index d339f5dff6..9204cdc523 100644 --- a/tests/v3/test_properties.py +++ b/tests/v3/test_properties.py @@ -7,7 +7,7 @@ import hypothesis.extra.numpy as npst # noqa import hypothesis.strategies as st # noqa from hypothesis import given, settings # noqa -from zarr.strategies import arrays, np_arrays, basic_indices # noqa +from zarr.testing.strategies import arrays, np_arrays, basic_indices # noqa @given(st.data()) diff --git a/tests/v3/test_store/test_stateful_store.py b/tests/v3/test_store/test_stateful_store.py new file mode 100644 index 0000000000..68bd11bbe8 --- /dev/null +++ b/tests/v3/test_store/test_stateful_store.py @@ -0,0 +1,229 @@ +# Stateful tests for arbitrary Zarr stores. + + +import hypothesis.strategies as st +from hypothesis import assume, note +from hypothesis.stateful import ( + RuleBasedStateMachine, + invariant, + precondition, + rule, +) + +import zarr +from zarr.abc.store import AccessMode, Store +from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype +from zarr.store import MemoryStore +from zarr.testing.strategies import key_ranges, paths + + +class SyncStoreWrapper(zarr.core.sync.SyncMixin): + def __init__(self, store: Store): + """Synchronous Store wrapper + + This class holds synchronous methods that map to async methods of Store classes. + The synchronous wrapper is needed because hypothesis' stateful testing infra does + not support asyncio so we redefine sync versions of the Store API. + https://github.com/HypothesisWorks/hypothesis/issues/3712#issuecomment-1668999041 + """ + self.store = store + + @property + def mode(self) -> AccessMode: + return self.store.mode + + def set(self, key: str, data_buffer: zarr.core.buffer.Buffer) -> None: + return self._sync(self.store.set(key, data_buffer)) + + def list(self) -> list: + return self._sync_iter(self.store.list()) + + def get(self, key: str, prototype: BufferPrototype) -> zarr.core.buffer.Buffer: + obs = self._sync(self.store.get(key, prototype=prototype)) + return obs + + def get_partial_values( + self, key_ranges: list, prototype: BufferPrototype + ) -> zarr.core.buffer.Buffer: + obs_partial = self._sync( + self.store.get_partial_values(prototype=prototype, key_ranges=key_ranges) + ) + return obs_partial + + def delete(self, path: str) -> None: + return self._sync(self.store.delete(path)) + + def empty(self) -> bool: + return self._sync(self.store.empty()) + + def clear(self) -> None: + return self._sync(self.store.clear()) + + def exists(self, key) -> bool: + return self._sync(self.store.exists(key)) + + def list_dir(self, prefix): + raise NotImplementedError + + def list_prefix(self, prefix: str): + raise NotImplementedError + + def set_partial_values(self, key_start_values): + raise NotImplementedError + + @property + def supports_listing(self) -> bool: + return self.store.supports_listing + + @property + def supports_partial_writes(self) -> bool: + return self.supports_partial_writes + + @property + def supports_writes(self) -> bool: + return self.store.supports_writes + + +class ZarrStoreStateMachine(RuleBasedStateMachine): + """ " + Zarr store state machine + + This is a subclass of a Hypothesis RuleBasedStateMachine. + It is testing a framework to ensure that the state of a Zarr store matches + an expected state after a set of random operations. It contains a store + (currently, a Zarr MemoryStore) and a model, a simplified version of a + zarr store (in this case, a dict). It also contains rules which represent + actions that can be applied to a zarr store. Rules apply an action to both + the store and the model, and invariants assert that the state of the model + is equal to the state of the store. Hypothesis then generates sequences of + rules, running invariants after each rule. It raises an error if a sequence + produces discontinuity between state of the model and state of the store + (ie. an invariant is violated). + https://hypothesis.readthedocs.io/en/latest/stateful.html + """ + + def __init__(self): + super().__init__() + self.model: dict[str, bytes] = {} + self.store = SyncStoreWrapper(MemoryStore(mode="w")) + self.prototype = default_buffer_prototype() + + @rule(key=paths, data=st.binary(min_size=0, max_size=100)) + def set(self, key: str, data: bytes) -> None: + note(f"(set) Setting {key!r} with {data}") + assert not self.store.mode.readonly + data_buf = Buffer.from_bytes(data) + self.store.set(key, data_buf) + self.model[key] = data_buf + + @precondition(lambda self: len(self.model.keys()) > 0) + @rule(key=paths, data=st.data()) + def get(self, key: str, data: bytes) -> None: + key = data.draw( + st.sampled_from(sorted(self.model.keys())) + ) # hypothesis wants to sample from sorted list + note("(get)") + store_value = self.store.get(key, self.prototype) + # to bytes here necessary because data_buf set to model in set() + assert self.model[key].to_bytes() == (store_value.to_bytes()) + + @rule(key=paths, data=st.data()) + def get_invalid_keys(self, key: str, data: bytes) -> None: + note("(get_invalid)") + assume(key not in self.model.keys()) + assert self.store.get(key, self.prototype) is None + + @precondition(lambda self: len(self.model.keys()) > 0) + @rule(data=st.data()) + def get_partial_values(self, data: bytes) -> None: + key_range = data.draw(key_ranges(keys=st.sampled_from(sorted(self.model.keys())))) + note(f"(get partial) {key_range=}") + obs_maybe = self.store.get_partial_values(key_range, self.prototype) + observed = [] + + for obs in obs_maybe: + assert obs is not None + observed.append(obs.to_bytes()) + + model_vals_ls = [] + + for key, byte_range in key_range: + start = byte_range[0] or 0 + step = byte_range[1] + stop = start + step if step is not None else None + model_vals_ls.append(self.model[key][start:stop]) + + assert all( + obs == exp.to_bytes() for obs, exp in zip(observed, model_vals_ls, strict=True) + ), ( + observed, + model_vals_ls, + ) + + @precondition(lambda self: len(self.model.keys()) > 0) + @rule(data=st.data()) + def delete(self, data: bytes) -> None: + key = data.draw(st.sampled_from(sorted(self.model.keys()))) + note(f"(delete) Deleting {key=}") + + self.store.delete(key) + del self.model[key] + + @rule() + def clear(self): + assert not self.store.mode.readonly + note("(clear)") + self.store.clear() + self.model.clear() + + assert len(self.model.keys()) == len(list(self.store.list())) == 0 + + @rule() + def empty(self) -> None: + note("(empty)") + + # make sure they either both are or both aren't empty (same state) + assert self.store.empty() == (not self.model) + + @rule(key=paths) + def exists(self, key: str) -> None: + note("(exists)") + + assert self.store.exists(key) == (key in self.model) + + @invariant() + def check_paths_equal(self) -> None: + note("Checking that paths are equal") + paths = list(self.store.list()) + + assert list(self.model.keys()) == paths + + @invariant() + def check_vals_equal(self) -> None: + note("Checking values equal") + for key, _val in self.model.items(): + store_item = self.store.get(key, self.prototype).to_bytes() + assert self.model[key].to_bytes() == store_item + + @invariant() + def check_num_keys_equal(self) -> None: + note("check num keys equal") + + assert len(self.model) == len(list(self.store.list())) + + @invariant() + def check_keys(self) -> None: + keys = list(self.store.list()) + + if len(keys) == 0: + assert self.store.empty() is True + + elif len(keys) != 0: + assert self.store.empty() is False + + for key in keys: + assert self.store.exists(key) is True + note("checking keys / exists / empty") + + +StatefulStoreTest = ZarrStoreStateMachine.TestCase From ee43bb861c1910879c5de5a01e24cc2c2516def4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:37:03 +0100 Subject: [PATCH 0114/1043] Bump the requirements group across 1 directory with 7 updates (#2092) Bumps the requirements group with 7 updates in the / directory: | Package | From | To | | --- | --- | --- | | [numpy](https://github.com/numpy/numpy) | `1.26.4` | `2.0.1` | | [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) | `8.1.2` | `8.1.3` | | [setuptools-scm](https://github.com/pypa/setuptools_scm) | `8.0.4` | `8.1.0` | | [pytest](https://github.com/pytest-dev/pytest) | `8.1.1` | `8.3.2` | | [lmdb](https://github.com/jnwatson/py-lmdb) | `1.4.1` | `1.5.1` | | [redis](https://github.com/redis/redis-py) | `5.0.4` | `5.0.8` | | [pymongo](https://github.com/mongodb/mongo-python-driver) | `4.6.3` | `4.8.0` | Updates `numpy` from 1.26.4 to 2.0.1 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v1.26.4...v2.0.1) Updates `ipywidgets` from 8.1.2 to 8.1.3 - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.2...8.1.3) Updates `setuptools-scm` from 8.0.4 to 8.1.0 - [Release notes](https://github.com/pypa/setuptools_scm/releases) - [Changelog](https://github.com/pypa/setuptools_scm/blob/main/CHANGELOG.md) - [Commits](https://github.com/pypa/setuptools_scm/compare/v8.0.4...v8.1.0) Updates `pytest` from 8.1.1 to 8.3.2 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.3.2) Updates `lmdb` from 1.4.1 to 1.5.1 - [Changelog](https://github.com/jnwatson/py-lmdb/blob/master/ChangeLog) - [Commits](https://github.com/jnwatson/py-lmdb/compare/py-lmdb_1.4.1...py-lmdb_1.5.1) Updates `redis` from 5.0.4 to 5.0.8 - [Release notes](https://github.com/redis/redis-py/releases) - [Changelog](https://github.com/redis/redis-py/blob/master/CHANGES) - [Commits](https://github.com/redis/redis-py/compare/v5.0.4...v5.0.8) Updates `pymongo` from 4.6.3 to 4.8.0 - [Release notes](https://github.com/mongodb/mongo-python-driver/releases) - [Changelog](https://github.com/mongodb/mongo-python-driver/blob/master/doc/changelog.rst) - [Commits](https://github.com/mongodb/mongo-python-driver/compare/4.6.3...4.8.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-major dependency-group: requirements - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: setuptools-scm dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: lmdb dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: redis dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: pymongo dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_minimal.txt | 4 ++-- requirements_dev_numpy.txt | 2 +- requirements_dev_optional.txt | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 5d156db655..df96c608b6 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -3,6 +3,6 @@ asciitree==0.3.3 fasteners==0.19 numcodecs==0.12.1 msgpack-python==0.5.6 -setuptools-scm==8.0.4 +setuptools-scm==8.1.0 # test requirements -pytest==8.1.1 +pytest==8.3.2 diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index d8d6c3d097..d53ec456a7 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==1.26.4 +numpy==2.0.1 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 3456cca21a..87c9538a3c 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -1,17 +1,17 @@ # optional library requirements # bsddb3==6.2.6; sys_platform != 'win32' -lmdb==1.4.1; sys_platform != 'win32' +lmdb==1.5.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.2 +ipywidgets==8.1.3 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) azure-storage-blob==12.16.0 # pyup: ignore -redis==5.0.4 +redis==5.0.8 types-redis types-setuptools -pymongo==4.6.3 +pymongo==4.8.0 # optional test requirements coverage pytest-cov==5.0.0 From addc39dfc53fd0f992f59196f91e78b81395f494 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:37:27 +0100 Subject: [PATCH 0115/1043] Bump the actions group with 2 updates (#2087) Bumps the actions group with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `actions/setup-python` from 5.1.0 to 5.1.1 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) Updates `pypa/gh-action-pypi-publish` from 1.8.14 to 1.9.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.8.14...v1.9.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 8ac76c899b..20cb5890b4 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.1.0 + - uses: actions/setup-python@v5.1.1 name: Install Python with: python-version: '3.9' @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.8.14 + - uses: pypa/gh-action-pypi-publish@v1.9.0 with: user: __token__ password: ${{ secrets.pypi_password }} From c5c4698baed2f74a545c163950ce8968306cfc99 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 17 Aug 2024 08:50:21 +0100 Subject: [PATCH 0116/1043] [v2] Drop support for Python 3.9 (#2074) * Drop support for Python 3.9 * Ignore B905 ruff rule * Fix release notes --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/python-package.yml | 2 +- .github/workflows/releases.yml | 2 +- .readthedocs.yaml | 2 +- docs/release.rst | 11 +++++++++-- pyproject.toml | 5 +++-- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index ec98af029e..7b0c4dcfc4 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -27,7 +27,7 @@ body: attributes: label: Python Version description: Version of Python interpreter - placeholder: 3.9, 3.10, 3.11, etc. + placeholder: 3.10, 3.11, 3.12 etc. validations: required: true - type: input diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f53cb2d9a9..61124b8e8e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - python-version: '3.10' diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 20cb5890b4..ec0601f0f4 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/setup-python@v5.1.1 name: Install Python with: - python-version: '3.9' + python-version: '3.11' - name: Install PyBuild run: | diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e45cae1b45..d7190b4771 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,7 +3,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" sphinx: configuration: docs/conf.py diff --git a/docs/release.rst b/docs/release.rst index c40ff5c0d3..9c2840a70a 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,7 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. + .. _release_2.18.3: 2.18.3 @@ -25,17 +26,23 @@ Release notes Maintenance ~~~~~~~~~~~ +* Removed support for Python 3.9. + By :user:`David Stansby ` + * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` .. _release_2.18.2: +2.18.2 +------ + Enhancements ~~~~~~~~~~~~ * Add Zstd codec to old V3 code path. - By :user:`Ryan Abernathey ` + By :user:`Ryan Abernathey ` .. _release_2.18.1: @@ -62,7 +69,7 @@ Maintenance * Enable ruff/bugbear rules (B) and fix issues. By :user:`Dimitri Papadopoulos Orfanos ` :issue:`1702`. -* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. +* Minor updates to use `np.inf` instead of `np.PINF` / `np.NINF` in preparation for NumPy 2.0.0 release. By :user:`Joe Hamman ` :issue:`1842`. Deprecations diff --git a/pyproject.toml b/pyproject.toml index a5668d0d90..da5fb34d89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ readme = { file = "README.md", content-type = "text/markdown" } maintainers = [ { name = "Alistair Miles", email = "alimanfoo@googlemail.com" } ] -requires-python = ">=3.9" +requires-python = ">=3.10" dependencies = [ 'asciitree', 'numpy>=1.23', @@ -30,9 +30,9 @@ classifiers = [ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', ] license = { text = "MIT" } @@ -107,6 +107,7 @@ exclude = [ extend-select = [ "B" ] +ignore = ["B905"] # zip-without-explicit-strict [tool.black] line-length = 100 From 35e8106278e94edd91dad370bab33f5633167044 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 23 Aug 2024 12:02:29 +0100 Subject: [PATCH 0117/1043] Fix Array.__array__ for numpy 2.1 (#2106) * Fix Array.__array__ for numpy 2.1 * Add changelog * Depend on np.array for array coercions --- docs/release.rst | 11 ++++++++++- zarr/core.py | 7 ++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 9c2840a70a..697dfde3b8 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -24,14 +24,23 @@ Release notes 2.18.3 ------ +Enhancements +~~~~~~~~~~~~ +* Added support for creating a copy of data when converting a `zarr.Array` + to a numpy array. + By :user:`David Stansby ` + Maintenance ~~~~~~~~~~~ * Removed support for Python 3.9. By :user:`David Stansby ` - + * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` +* Added compatibility with numpy 2.1. + By :user:`David Stansby ` + .. _release_2.18.2: diff --git a/zarr/core.py b/zarr/core.py index 141190a0c8..08234e193c 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -575,11 +575,8 @@ def __eq__(self, other): # store comparison ) - def __array__(self, *args): - a = self[...] - if args: - a = a.astype(args[0]) - return a + def __array__(self, dtype=None, copy=None): + return np.array(self[...], dtype=dtype, copy=copy) def islice(self, start=None, end=None): """ From 025e1031fc43b6e672b0dc9a53953afb1b9eef5d Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 26 Aug 2024 16:57:00 +0100 Subject: [PATCH 0118/1043] Bump test version of numcodecs (#2114) * Bump test version of numcodecs * Fix test for numcodecs 0.13 --- requirements_dev_minimal.txt | 2 +- zarr/tests/test_meta.py | 34 +++++++++++++--------------------- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index df96c608b6..2991d99299 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -1,7 +1,7 @@ # library requirements asciitree==0.3.3 fasteners==0.19 -numcodecs==0.12.1 +numcodecs==0.13.0 msgpack-python==0.5.6 setuptools-scm==8.1.0 # test requirements diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index 7b7d526476..57ab9a0781 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -282,36 +282,28 @@ def test_encode_decode_array_dtype_shape_v3(cname): fill_value=None, chunk_memory_layout="C", ) - - meta_json = ( - """{ + meta_expected = { "attributes": {}, - "chunk_grid": { - "chunk_shape": [10], - "separator": "/", - "type": "regular" - }, + "chunk_grid": {"chunk_shape": [10], "separator": "/", "type": "regular"}, "chunk_memory_layout": "C", "compressor": { - """ - + f""" - "codec": "https://purl.org/zarr/spec/codec/{cname}/1.0", - """ - + """ - "configuration": { - "level": 1 - } + "codec": f"https://purl.org/zarr/spec/codec/{cname}/1.0", + "configuration": {"level": 1}, }, "data_type": " Date: Mon, 26 Aug 2024 12:56:20 -0700 Subject: [PATCH 0119/1043] fix: numpy 1.24 compat for Array.__array__ (#2123) --- zarr/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zarr/core.py b/zarr/core.py index 08234e193c..d13da27bc6 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -575,8 +575,8 @@ def __eq__(self, other): # store comparison ) - def __array__(self, dtype=None, copy=None): - return np.array(self[...], dtype=dtype, copy=copy) + def __array__(self, *args, **kwargs): + return np.array(self[...], *args, **kwargs) def islice(self, start=None, end=None): """ From 61683be6f13775c3b9fa78b433812ec1fac41786 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 26 Aug 2024 15:06:47 -0500 Subject: [PATCH 0120/1043] Fixed MemoryStore.list_dir (#2117) * Fixed MemoryStore.list_dir Ensures that nested children are listed properly. * fixup s3 * simplify * Update src/zarr/testing/store.py Co-authored-by: David Stansby * fixup tests --------- Co-authored-by: David Stansby --- src/zarr/store/memory.py | 16 +++++++++++----- src/zarr/store/remote.py | 2 +- src/zarr/testing/store.py | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 999d750755..117fd69ec0 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -114,9 +114,15 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: if prefix == "": keys_unique = set(k.split("/")[0] for k in self._store_dict.keys()) - for key in keys_unique: - yield key else: - for key in self._store_dict: - if key.startswith(prefix + "/") and key != prefix: - yield key.removeprefix(prefix + "/").split("/")[0] + # Our dictionary doesn't contain directory markers, but we want to include + # a pseudo directory when there's a nested item and we're listing an + # intermediate level. + keys_unique = { + key.removeprefix(prefix + "/").split("/")[0] + for key in self._store_dict + if key.startswith(prefix + "/") and key != prefix + } + + for key in keys_unique: + yield key diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index f5ea694b0a..83393e4dac 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -202,7 +202,7 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: except FileNotFoundError: return for onefile in (a.replace(prefix + "/", "") for a in allfiles): - yield onefile + yield onefile.removeprefix(self.path).removeprefix("/") async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: for onefile in await self._fs._ls(prefix, detail=False): diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index ebef4824f7..11978b4121 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -191,9 +191,17 @@ async def test_list_dir(self, store: S) -> None: assert out == [] assert [k async for k in store.list_dir("foo")] == [] await store.set("foo/zarr.json", Buffer.from_bytes(b"bar")) - await store.set("foo/c/1", Buffer.from_bytes(b"\x01")) + await store.set("group-0/zarr.json", Buffer.from_bytes(b"\x01")) # group + await store.set("group-0/group-1/zarr.json", Buffer.from_bytes(b"\x01")) # group + await store.set("group-0/group-1/a1/zarr.json", Buffer.from_bytes(b"\x01")) + await store.set("group-0/group-1/a2/zarr.json", Buffer.from_bytes(b"\x01")) + await store.set("group-0/group-1/a3/zarr.json", Buffer.from_bytes(b"\x01")) - keys_expected = ["zarr.json", "c"] + keys_expected = ["foo", "group-0"] + keys_observed = [k async for k in store.list_dir("")] + assert set(keys_observed) == set(keys_expected) + + keys_expected = ["zarr.json"] keys_observed = [k async for k in store.list_dir("foo")] assert len(keys_observed) == len(keys_expected), keys_observed @@ -202,3 +210,23 @@ async def test_list_dir(self, store: S) -> None: keys_observed = [k async for k in store.list_dir("foo/")] assert len(keys_expected) == len(keys_observed), keys_observed assert set(keys_observed) == set(keys_expected), keys_observed + + keys_observed = [k async for k in store.list_dir("group-0")] + keys_expected = ["zarr.json", "group-1"] + + assert len(keys_observed) == len(keys_expected), keys_observed + assert set(keys_observed) == set(keys_expected), keys_observed + + keys_observed = [k async for k in store.list_dir("group-0/")] + assert len(keys_expected) == len(keys_observed), keys_observed + assert set(keys_observed) == set(keys_expected), keys_observed + + keys_observed = [k async for k in store.list_dir("group-0/group-1")] + keys_expected = ["zarr.json", "a1", "a2", "a3"] + + assert len(keys_observed) == len(keys_expected), keys_observed + assert set(keys_observed) == set(keys_expected), keys_observed + + keys_observed = [k async for k in store.list_dir("group-0/group-1")] + assert len(keys_expected) == len(keys_observed), keys_observed + assert set(keys_observed) == set(keys_expected), keys_observed From 2e928b145c5ffdb088c9973641d71ac2937e5e49 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 26 Aug 2024 14:32:22 -0700 Subject: [PATCH 0121/1043] Deprecate N5Store (#2103) * deprecate(n5): add deprecation warning to N5Store * also deprecate N5FSStore * docs * fix doc --- docs/release.rst | 11 +++++++++-- zarr/n5.py | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 697dfde3b8..05110f8ce1 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -28,12 +28,13 @@ Enhancements ~~~~~~~~~~~~ * Added support for creating a copy of data when converting a `zarr.Array` to a numpy array. - By :user:`David Stansby ` + By :user:`David Stansby ` (:issue:`2106`) and + :user:`Joe Hamman ` (:issue:`2123`). Maintenance ~~~~~~~~~~~ * Removed support for Python 3.9. - By :user:`David Stansby ` + By :user:`David Stansby ` (:issue:`2074`). * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` @@ -41,6 +42,12 @@ Maintenance * Added compatibility with numpy 2.1. By :user:`David Stansby ` +Deprecations +~~~~~~~~~~~~ + +* Deprecate :class:`zarr.n5.N5Store` and :class:`zarr.n5.N5FSStore`. These + stores are slated to be removed in Zarr Python 3.0. + By :user:`Joe Hamman ` :issue:`2085`. .. _release_2.18.2: diff --git a/zarr/n5.py b/zarr/n5.py index 3d3e9afa26..3bb7093128 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -69,8 +69,20 @@ class N5Store(NestedDirectoryStore): Safe to write in multiple threads or processes. + .. deprecated:: 2.18.3 + `N5Store` will be removed in Zarr 3.0.0. """ + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + warnings.warn( + "The N5Store is deprecated and will be removed in a Zarr-Python version 3, " + "see https://github.com/zarr-developers/zarr-python/issues/1274 and " + "https://github.com/zarr-developers/n5py for more information.", + FutureWarning, + stacklevel=2, + ) + def __getitem__(self, key: str) -> bytes: if key.endswith(zarr_group_meta_key): key_new = key.replace(zarr_group_meta_key, n5_attrs_key) @@ -322,6 +334,9 @@ class N5FSStore(FSStore): storage, and this procedure requires chunk keys with "." separated dimensions, hence the Zarr arrays targeting N5 have the deceptive "." dimension separator. + + .. deprecated:: 2.18.3 + `N5FSStore` will be removed in Zarr 3.0.0. """ _array_meta_key = "attributes.json" @@ -329,6 +344,13 @@ class N5FSStore(FSStore): _attrs_key = "attributes.json" def __init__(self, *args, **kwargs): + warnings.warn( + "The N5FSStore is deprecated and will be removed in a Zarr-Python version 3, " + "see https://github.com/zarr-developers/zarr-python/issues/1274 and " + "https://github.com/zarr-developers/n5py for more information.", + FutureWarning, + stacklevel=2, + ) if "dimension_separator" in kwargs: kwargs.pop("dimension_separator") warnings.warn( From 5bf57bd7492159823f83937fc7ccab2fdb61eeb5 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Mon, 26 Aug 2024 23:31:38 +0100 Subject: [PATCH 0122/1043] Run tests on numpy 1.23 (#2124) --- .github/workflows/python-package.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 61124b8e8e..f50fa7d9e0 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,8 +18,6 @@ jobs: python-version: ['3.10', '3.11', '3.12'] numpy_version: ['>=1.24.0', '==1.23.*'] exclude: - - python-version: '3.10' - numpy_version: '==1.23.*' - python-version: '3.11' numpy_version: '==1.23.*' - python-version: '3.12' From 9e0b5e88d9473fb3403a9cb35eff0526a75197cb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:37:09 +0100 Subject: [PATCH 0123/1043] Bump the requirements group across 1 directory with 3 updates (#2129) Bumps the requirements group with 3 updates in the / directory: [numpy](https://github.com/numpy/numpy), [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) and [azure-storage-blob](https://github.com/Azure/azure-sdk-for-python). Updates `numpy` from 2.0.1 to 2.1.0 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.0.1...v2.1.0) Updates `ipywidgets` from 8.1.3 to 8.1.5 - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/8.1.3...8.1.5) Updates `azure-storage-blob` from 12.16.0 to 12.21.0 - [Release notes](https://github.com/Azure/azure-sdk-for-python/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/esrp_release.md) - [Commits](https://github.com/Azure/azure-sdk-for-python/compare/azure-storage-blob_12.16.0...azure-storage-blob_12.21.0) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements - dependency-name: ipywidgets dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements - dependency-name: azure-storage-blob dependency-type: direct:development update-type: version-update:semver-minor dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_numpy.txt | 2 +- requirements_dev_optional.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index d53ec456a7..6423997a29 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==2.0.1 +numpy==2.1.0 diff --git a/requirements_dev_optional.txt b/requirements_dev_optional.txt index 87c9538a3c..9d75d78e44 100644 --- a/requirements_dev_optional.txt +++ b/requirements_dev_optional.txt @@ -3,11 +3,11 @@ lmdb==1.5.1; sys_platform != 'win32' # optional library requirements for Jupyter ipytree==0.2.2 -ipywidgets==8.1.3 +ipywidgets==8.1.5 # optional library requirements for services # don't let pyup change pinning for azure-storage-blob, need to pin to older # version to get compatibility with azure storage emulator on appveyor (FIXME) -azure-storage-blob==12.16.0 # pyup: ignore +azure-storage-blob==12.21.0 # pyup: ignore redis==5.0.8 types-redis types-setuptools From cc2cdeeccca4a0db2adf4bcde0b7f91b2100bbec Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 27 Aug 2024 10:10:37 -0700 Subject: [PATCH 0124/1043] chore: bump minimum numpy version to 1.24 (#2127) * chore: bump minimum version to 1.24 * Update .github/workflows/python-package.yml --- .github/workflows/python-package.yml | 8 +++----- docs/release.rst | 5 ++++- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f50fa7d9e0..76ae9f982e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,12 +16,10 @@ jobs: strategy: matrix: python-version: ['3.10', '3.11', '3.12'] - numpy_version: ['>=1.24.0', '==1.23.*'] + numpy_version: ['>=2.1', '==1.24.*'] exclude: - - python-version: '3.11' - numpy_version: '==1.23.*' - python-version: '3.12' - numpy_version: '==1.23.*' + numpy_version: '==1.24.*' services: redis: image: redis @@ -61,7 +59,7 @@ jobs: conda activate zarr-env python -m pip install --upgrade pip python -m pip install -U pip setuptools wheel line_profiler - python -m pip install -rrequirements_dev_minimal.txt numpy${{matrix.numpy_version}} -rrequirements_dev_optional.txt pymongo redis + python -m pip install -r requirements_dev_minimal.txt numpy${{matrix.numpy_version}} -r requirements_dev_optional.txt pymongo redis python -m pip install -e . python -m pip freeze - name: Tests diff --git a/docs/release.rst b/docs/release.rst index 05110f8ce1..69616d6052 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -39,9 +39,12 @@ Maintenance * Fix a regression when using orthogonal indexing with a scalar. By :user:`Deepak Cherian ` :issue:`1931` -* Added compatibility with numpy 2.1. +* Added compatibility with NumPy 2.1. By :user:`David Stansby ` +* Bump minimum NumPy version to 1.24. + :user:`Joe Hamman ` (:issue:`2127`). + Deprecations ~~~~~~~~~~~~ diff --git a/pyproject.toml b/pyproject.toml index da5fb34d89..ec06b63a96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ maintainers = [ requires-python = ">=3.10" dependencies = [ 'asciitree', - 'numpy>=1.23', + 'numpy>=1.24', 'fasteners; sys_platform != "emscripten"', 'numcodecs>=0.10.0', ] From 0095ee62b662870b470eec1aa0da44f0b918a476 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 29 Aug 2024 06:25:43 -0500 Subject: [PATCH 0125/1043] Feature/recursive members (#2118) * Fixed MemoryStore.list_dir Ensures that nested children are listed properly. * fixup s3 * recursive Group.members This PR adds a recursive=True flag to Group.members, for recursively listing the members of some hierarhcy. This is useful for Consolidated Metadata, which needs to recursively inspect children. IMO, it's useful (and simple) enough to include in the public API. * trigger ci * fixed datetime serialization * fixup * fixed invalid escape sequence * fixup * max_depth * max_depth=None --- src/zarr/core/group.py | 70 +++++++++++++++++++++++---- src/zarr/core/metadata.py | 22 ++++++--- src/zarr/testing/strategies.py | 10 +++- tests/v3/test_group.py | 78 +++++++++++++++++++++++++++++-- tests/v3/test_metadata/test_v3.py | 23 +++++++++ tests/v3/test_properties.py | 10 ++++ 6 files changed, 191 insertions(+), 22 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 86d27e3a97..2c26cac3b1 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -424,21 +424,59 @@ async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: def __repr__(self) -> str: return f"" - async def nmembers(self) -> int: + async def nmembers( + self, + max_depth: int | None = 0, + ) -> int: + """ + Count the number of members in this group. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + + Returns + ------- + count : int + """ # TODO: consider using aioitertools.builtins.sum for this # return await aioitertools.builtins.sum((1 async for _ in self.members()), start=0) n = 0 - async for _ in self.members(): + async for _ in self.members(max_depth=max_depth): n += 1 return n - async def members(self) -> AsyncGenerator[tuple[str, AsyncArray | AsyncGroup], None]: + async def members( + self, + max_depth: int | None = 0, + ) -> AsyncGenerator[tuple[str, AsyncArray | AsyncGroup], None]: """ Returns an AsyncGenerator over the arrays and groups contained in this group. This method requires that `store_path.store` supports directory listing. The results are not guaranteed to be ordered. + + Parameters + ---------- + max_depth : int, default 0 + The maximum number of levels of the hierarchy to include. By + default, (``max_depth=0``) only immediate children are included. Set + ``max_depth=None`` to include all nodes, and some positive integer + to consider children within that many levels of the root Group. + """ + if max_depth is not None and max_depth < 0: + raise ValueError(f"max_depth must be None or >= 0. Got '{max_depth}' instead") + async for item in self._members(max_depth=max_depth, current_depth=0): + yield item + + async def _members( + self, max_depth: int | None, current_depth: int + ) -> AsyncGenerator[tuple[str, AsyncArray | AsyncGroup], None]: if not self.store_path.store.supports_listing: msg = ( f"The store associated with this group ({type(self.store_path.store)}) " @@ -456,7 +494,21 @@ async def members(self) -> AsyncGenerator[tuple[str, AsyncArray | AsyncGroup], N if key in _skip_keys: continue try: - yield (key, await self.getitem(key)) + obj = await self.getitem(key) + yield (key, obj) + + if ( + ((max_depth is None) or (current_depth < max_depth)) + and hasattr(obj.metadata, "node_type") + and obj.metadata.node_type == "group" + ): + # the assert is just for mypy to know that `obj.metadata.node_type` + # implies an AsyncGroup, not an AsyncArray + assert isinstance(obj, AsyncGroup) + async for child_key, val in obj._members( + max_depth=max_depth, current_depth=current_depth + 1 + ): + yield "/".join([key, child_key]), val except KeyError: # keyerror is raised when `key` names an object (in the object storage sense), # as opposed to a prefix, in the store under the prefix associated with this group @@ -628,17 +680,15 @@ def update_attributes(self, new_attributes: dict[str, Any]) -> Group: self._sync(self._async_group.update_attributes(new_attributes)) return self - @property - def nmembers(self) -> int: - return self._sync(self._async_group.nmembers()) + def nmembers(self, max_depth: int | None = 0) -> int: + return self._sync(self._async_group.nmembers(max_depth=max_depth)) - @property - def members(self) -> tuple[tuple[str, Array | Group], ...]: + def members(self, max_depth: int | None = 0) -> tuple[tuple[str, Array | Group], ...]: """ Return the sub-arrays and sub-groups of this group as a tuple of (name, array | group) pairs """ - _members = self._sync_iter(self._async_group.members()) + _members = self._sync_iter(self._async_group.members(max_depth=max_depth)) result = tuple(map(lambda kv: (kv[0], _parse_async_node(kv[1])), _members)) return result diff --git a/src/zarr/core/metadata.py b/src/zarr/core/metadata.py index d541e43205..72172a2673 100644 --- a/src/zarr/core/metadata.py +++ b/src/zarr/core/metadata.py @@ -256,13 +256,21 @@ def _json_convert(o: Any) -> Any: if isinstance(o, np.dtype): return str(o) if np.isscalar(o): - # convert numpy scalar to python type, and pass - # python types through - out = getattr(o, "item", lambda: o)() - if isinstance(out, complex): - # python complex types are not JSON serializable, so we use the - # serialization defined in the zarr v3 spec - return [out.real, out.imag] + out: Any + if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): + # https://github.com/zarr-developers/zarr-python/issues/2119 + # `.item()` on a datetime type might or might not return an + # integer, depending on the value. + # Explicitly cast to an int first, and then grab .item() + out = o.view("i8").item() + else: + # convert numpy scalar to python type, and pass + # python types through + out = getattr(o, "item", lambda: o)() + if isinstance(out, complex): + # python complex types are not JSON serializable, so we use the + # serialization defined in the zarr v3 spec + return [out.real, out.imag] return out if isinstance(o, Enum): return o.name diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index d2e41c6290..3a460d4fff 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -1,3 +1,4 @@ +import re from typing import Any import hypothesis.extra.numpy as npst @@ -101,7 +102,14 @@ def arrays( root = Group.create(store) fill_value_args: tuple[Any, ...] = tuple() if nparray.dtype.kind == "M": - fill_value_args = ("ns",) + m = re.search(r"\[(.+)\]", nparray.dtype.str) + if not m: + raise ValueError(f"Couldn't find precision for dtype '{nparray.dtype}.") + + fill_value_args = ( + # e.g. ns, D + m.groups()[0], + ) a = root.create_array( array_path, diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 39921c26d8..eb7b1f30dd 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -88,7 +88,8 @@ def test_group_members(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) members_expected["subgroup"] = group.create_group("subgroup") # make a sub-sub-subgroup, to ensure that the children calculation doesn't go # too deep in the hierarchy - _ = members_expected["subgroup"].create_group("subsubgroup") # type: ignore + subsubgroup = members_expected["subgroup"].create_group("subsubgroup") # type: ignore + subsubsubgroup = subsubgroup.create_group("subsubsubgroup") # type: ignore members_expected["subarray"] = group.create_array( "subarray", shape=(100,), dtype="uint8", chunk_shape=(10,), exists_ok=True @@ -101,10 +102,25 @@ def test_group_members(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) # this creates a directory with a random key in it # this should not show up as a member sync(store.set(f"{path}/extra_directory/extra_object-2", Buffer.from_bytes(b"000000"))) - members_observed = group.members + members_observed = group.members() # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) + # partial + members_observed = group.members(max_depth=1) + members_expected["subgroup/subsubgroup"] = subsubgroup + # members are not guaranteed to be ordered, so sort before comparing + assert sorted(dict(members_observed)) == sorted(members_expected) + + # total + members_observed = group.members(max_depth=None) + members_expected["subgroup/subsubgroup/subsubsubgroup"] = subsubsubgroup + # members are not guaranteed to be ordered, so sort before comparing + assert sorted(dict(members_observed)) == sorted(members_expected) + + with pytest.raises(ValueError, match="max_depth"): + members_observed = group.members(max_depth=-1) + def test_group(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: """ @@ -349,7 +365,8 @@ def test_group_create_array( if method == "create_array": array = group.create_array(name="array", shape=shape, dtype=dtype, data=data) elif method == "array": - array = group.array(name="array", shape=shape, dtype=dtype, data=data) + with pytest.warns(DeprecationWarning): + array = group.array(name="array", shape=shape, dtype=dtype, data=data) else: raise AssertionError @@ -358,7 +375,7 @@ def test_group_create_array( with pytest.raises(ContainsArrayError): group.create_array(name="array", shape=shape, dtype=dtype, data=data) elif method == "array": - with pytest.raises(ContainsArrayError): + with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): group.array(name="array", shape=shape, dtype=dtype, data=data) assert array.shape == shape assert array.dtype == np.dtype(dtype) @@ -653,3 +670,56 @@ async def test_asyncgroup_update_attributes( agroup_new_attributes = await agroup.update_attributes(attributes_new) assert agroup_new_attributes.attrs == attributes_new + + +async def test_group_members_async(store: LocalStore | MemoryStore): + group = AsyncGroup( + GroupMetadata(), + store_path=StorePath(store=store, path="root"), + ) + a0 = await group.create_array("a0", (1,)) + g0 = await group.create_group("g0") + a1 = await g0.create_array("a1", (1,)) + g1 = await g0.create_group("g1") + a2 = await g1.create_array("a2", (1,)) + g2 = await g1.create_group("g2") + + # immediate children + children = sorted([x async for x in group.members()], key=lambda x: x[0]) + assert children == [ + ("a0", a0), + ("g0", g0), + ] + + nmembers = await group.nmembers() + assert nmembers == 2 + + # partial + children = sorted([x async for x in group.members(max_depth=1)], key=lambda x: x[0]) + expected = [ + ("a0", a0), + ("g0", g0), + ("g0/a1", a1), + ("g0/g1", g1), + ] + assert children == expected + nmembers = await group.nmembers(max_depth=1) + assert nmembers == 4 + + # all children + all_children = sorted([x async for x in group.members(max_depth=None)], key=lambda x: x[0]) + expected = [ + ("a0", a0), + ("g0", g0), + ("g0/a1", a1), + ("g0/g1", g1), + ("g0/g1/a2", a2), + ("g0/g1/g2", g2), + ] + assert all_children == expected + + nmembers = await group.nmembers(max_depth=None) + assert nmembers == 6 + + with pytest.raises(ValueError, match="max_depth"): + [x async for x in group.members(max_depth=-1)] diff --git a/tests/v3/test_metadata/test_v3.py b/tests/v3/test_metadata/test_v3.py index eedcdf6234..1a0c5b94d7 100644 --- a/tests/v3/test_metadata/test_v3.py +++ b/tests/v3/test_metadata/test_v3.py @@ -1,10 +1,12 @@ from __future__ import annotations +import json import re from typing import TYPE_CHECKING, Literal from zarr.abc.codec import Codec from zarr.codecs.bytes import BytesCodec +from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding if TYPE_CHECKING: @@ -230,3 +232,24 @@ def test_metadata_to_dict( observed.pop("chunk_key_encoding") expected.pop("chunk_key_encoding") assert observed == expected + + +@pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) +@pytest.mark.parametrize("precision", ["ns", "D"]) +async def test_datetime_metadata(fill_value: int, precision: str): + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": f"> $GITHUB_PATH + - name: GPU check + run: | + nvidia-smi + echo $PATH + echo $LD_LIBRARY_PATH + nvcc -V + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install Hatch and CuPy + run: | + python -m pip install --upgrade pip + pip install hatch + - name: Set Up Hatch Env + run: | + hatch env create gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} + hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env + - name: Run Tests + run: | + hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage diff --git a/pyproject.toml b/pyproject.toml index 197c4aca10..8bee491a82 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,9 @@ jupyter = [ 'ipytree>=0.2.2', 'ipywidgets>=8.0.0', ] +gpu = [ + "cupy-cuda12x", +] docs = [ 'sphinx', 'sphinx-autobuild>=2021.3.14', @@ -120,7 +123,7 @@ build.hooks.vcs.version-file = "src/zarr/_version.py" [tool.hatch.envs.test] dependencies = [ "numpy~={matrix:numpy}", - "universal_pathlib" + "universal_pathlib", ] features = ["test", "extra"] @@ -134,8 +137,34 @@ python = ["3.10", "3.11", "3.12"] numpy = ["1.24", "1.26", "2.0"] features = ["optional"] +[[tool.hatch.envs.test.matrix]] +python = ["3.10", "3.11", "3.12"] +numpy = ["1.24", "1.26", "2.0"] +features = ["gpu"] + [tool.hatch.envs.test.scripts] run-coverage = "pytest --cov-config=pyproject.toml --cov=pkg --cov=tests" +run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests" +run = "run-coverage --no-cov" +run-verbose = "run-coverage --verbose" +run-mypy = "mypy src" +run-hypothesis = "pytest --hypothesis-profile ci tests/v3/test_properties.py tests/v3/test_store/test_stateful*" +list-env = "pip list" + +[tool.hatch.envs.gputest] +dependencies = [ + "numpy~={matrix:numpy}", + "universal_pathlib", +] +features = ["test", "extra", "gpu"] + +[[tool.hatch.envs.gputest.matrix]] +python = ["3.10", "3.11", "3.12"] +numpy = ["1.24", "1.26", "2.0"] +version = ["minimal"] + +[tool.hatch.envs.gputest.scripts] +run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov=tests" run = "run-coverage --no-cov" run-verbose = "run-coverage --verbose" run-mypy = "mypy src" @@ -223,4 +252,8 @@ filterwarnings = [ "error:::zarr.*", "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", + "ignore:Creating a zarr.buffer.gpu.*:UserWarning", +] +markers = [ + "gpu: mark a test as requiring CuPy and GPU" ] diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index 8e01524992..ffe5a0b6f3 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -10,7 +10,8 @@ from zarr.abc.codec import BytesBytesCodec from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import Buffer, as_numpy_array_wrapper +from zarr.core.buffer import Buffer +from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_enum, parse_named_configuration, to_thread from zarr.registry import register_codec diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py index bab81a104b..d8d793a965 100644 --- a/src/zarr/codecs/gzip.py +++ b/src/zarr/codecs/gzip.py @@ -7,7 +7,8 @@ from zarr.abc.codec import BytesBytesCodec from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import Buffer, as_numpy_array_wrapper +from zarr.core.buffer import Buffer +from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration, to_thread from zarr.registry import register_codec diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index 92a2665425..fceb49cf2b 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -9,7 +9,8 @@ from zarr.abc.codec import BytesBytesCodec from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import Buffer, as_numpy_array_wrapper +from zarr.core.buffer import Buffer +from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration, to_thread from zarr.registry import register_codec diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 3738b0859b..e7b5709b77 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -512,7 +512,12 @@ async def _set_selection( # check value shape if np.isscalar(value): - value = np.asanyarray(value, dtype=self.metadata.dtype) + array_like = prototype.buffer.create_zero_length().as_array_like() + if isinstance(array_like, np._typing._SupportsArrayFunc): + # TODO: need to handle array types that don't support __array_function__ + # like PyTorch and JAX + array_like_ = cast(np._typing._SupportsArrayFunc, array_like) + value = np.asanyarray(value, dtype=self.metadata.dtype, like=array_like_) else: if not hasattr(value, "shape"): value = np.asarray(value, self.metadata.dtype) @@ -520,7 +525,11 @@ async def _set_selection( # value.shape == indexer.shape # ), f"shape of value doesn't match indexer shape. Expected {indexer.shape}, got {value.shape}" if not hasattr(value, "dtype") or value.dtype.name != self.metadata.dtype.name: - value = np.array(value, dtype=self.metadata.dtype, order="A") + if hasattr(value, "astype"): + # Handle things that are already NDArrayLike more efficiently + value = value.astype(dtype=self.metadata.dtype, order="A") + else: + value = np.array(value, dtype=self.metadata.dtype, order="A") value = cast(NDArrayLike, value) # We accept any ndarray like object from the user and convert it # to a NDBuffer (or subclass). From this point onwards, we only pass diff --git a/src/zarr/core/buffer/__init__.py b/src/zarr/core/buffer/__init__.py new file mode 100644 index 0000000000..b18417b00b --- /dev/null +++ b/src/zarr/core/buffer/__init__.py @@ -0,0 +1,19 @@ +from zarr.core.buffer.core import ( + ArrayLike, + Buffer, + BufferPrototype, + NDArrayLike, + NDBuffer, + default_buffer_prototype, +) +from zarr.core.buffer.cpu import numpy_buffer_prototype + +__all__ = [ + "ArrayLike", + "Buffer", + "NDArrayLike", + "NDBuffer", + "BufferPrototype", + "default_buffer_prototype", + "numpy_buffer_prototype", +] diff --git a/src/zarr/core/buffer.py b/src/zarr/core/buffer/core.py similarity index 84% rename from src/zarr/core/buffer.py rename to src/zarr/core/buffer/core.py index 50252590dd..6e5d9465cf 100644 --- a/src/zarr/core/buffer.py +++ b/src/zarr/core/buffer/core.py @@ -1,7 +1,8 @@ from __future__ import annotations import sys -from collections.abc import Callable, Iterable, Sequence +from abc import ABC, abstractmethod +from collections.abc import Iterable, Sequence from typing import ( TYPE_CHECKING, Any, @@ -9,6 +10,7 @@ NamedTuple, Protocol, SupportsIndex, + cast, runtime_checkable, ) @@ -19,8 +21,6 @@ from zarr.registry import ( get_buffer_class, get_ndbuffer_class, - register_buffer, - register_ndbuffer, ) if TYPE_CHECKING: @@ -112,7 +112,7 @@ def check_item_key_is_1d_contiguous(key: Any) -> None: raise ValueError("slice must be contiguous") -class Buffer: +class Buffer(ABC): """A flat contiguous memory block We use Buffer throughout Zarr to represent a contiguous block of memory. @@ -141,6 +141,7 @@ def __init__(self, array_like: ArrayLike): self._data = array_like @classmethod + @abstractmethod def create_zero_length(cls) -> Self: """Create an empty buffer with length zero @@ -148,7 +149,11 @@ def create_zero_length(cls) -> Self: ------- New empty 0-length buffer """ - return cls(np.array([], dtype="b")) + if cls is Buffer: + raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") + return cls( + cast(ArrayLike, None) + ) # This line will never be reached, but it satisfies the type checker @classmethod def from_array_like(cls, array_like: ArrayLike) -> Self: @@ -166,6 +171,7 @@ def from_array_like(cls, array_like: ArrayLike) -> Self: return cls(array_like) @classmethod + @abstractmethod def from_buffer(cls, buffer: Buffer) -> Self: """Create a new buffer of an existing Buffer @@ -185,10 +191,20 @@ def from_buffer(cls, buffer: Buffer) -> Self: Returns ------- A new buffer representing the content of the input buffer + + Note + ---- + Subclasses of `Buffer` must override this method to implement + more optimal conversions that avoid copies where possible """ - return cls.from_array_like(buffer.as_array_like()) + if cls is Buffer: + raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") + return cls( + cast(ArrayLike, None) + ) # This line will never be reached, but it satisfies the type checker @classmethod + @abstractmethod def from_bytes(cls, bytes_like: BytesLike) -> Self: """Create a new buffer of a bytes-like object (host memory) @@ -201,7 +217,11 @@ def from_bytes(cls, bytes_like: BytesLike) -> Self: ------- New buffer representing `bytes_like` """ - return cls.from_array_like(np.frombuffer(bytes_like, dtype="b")) + if cls is Buffer: + raise NotImplementedError("Cannot call abstract method on the abstract class 'Buffer'") + return cls( + cast(ArrayLike, None) + ) # This line will never be reached, but it satisfies the type checker def as_array_like(self) -> ArrayLike: """Returns the underlying array (host or device memory) of this buffer @@ -214,6 +234,7 @@ def as_array_like(self) -> ArrayLike: """ return self._data + @abstractmethod def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). @@ -225,7 +246,7 @@ def as_numpy_array(self) -> npt.NDArray[Any]: ------- NumPy array of this buffer (might be a data copy) """ - return np.asanyarray(self._data) + ... def to_bytes(self) -> bytes: """Returns the buffer as `bytes` (host memory). @@ -252,14 +273,10 @@ def __setitem__(self, key: slice, value: Any) -> None: def __len__(self) -> int: return self._data.size + @abstractmethod def __add__(self, other: Buffer) -> Self: """Concatenate two buffers""" - - other_array = other.as_array_like() - assert other_array.dtype == np.dtype("b") - return self.__class__( - np.concatenate((np.asanyarray(self._data), np.asanyarray(other_array))) - ) + ... class NDBuffer: @@ -293,6 +310,7 @@ def __init__(self, array: NDArrayLike): self._data = array @classmethod + @abstractmethod def create( cls, *, @@ -324,10 +342,13 @@ def create( A subclass can overwrite this method to create a ndarray-like object other then the default Numpy array. """ - ret = cls(np.empty(shape=tuple(shape), dtype=dtype, order=order)) - if fill_value is not None: - ret.fill(fill_value) - return ret + if cls is NDBuffer: + raise NotImplementedError( + "Cannot call abstract method on the abstract class 'NDBuffer'" + ) + return cls( + cast(NDArrayLike, None) + ) # This line will never be reached, but it satisfies the type checker @classmethod def from_ndarray_like(cls, ndarray_like: NDArrayLike) -> Self: @@ -345,6 +366,7 @@ def from_ndarray_like(cls, ndarray_like: NDArrayLike) -> Self: return cls(ndarray_like) @classmethod + @abstractmethod def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: """Create a new buffer of Numpy array-like object @@ -357,7 +379,13 @@ def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: ------- New buffer representing `array_like` """ - return cls.from_ndarray_like(np.asanyarray(array_like)) + if cls is NDBuffer: + raise NotImplementedError( + "Cannot call abstract method on the abstract class 'NDBuffer'" + ) + return cls( + cast(NDArrayLike, None) + ) # This line will never be reached, but it satisfies the type checker def as_ndarray_like(self) -> NDArrayLike: """Returns the underlying array (host or device memory) of this buffer @@ -370,6 +398,7 @@ def as_ndarray_like(self) -> NDArrayLike: """ return self._data + @abstractmethod def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). @@ -381,7 +410,7 @@ def as_numpy_array(self) -> npt.NDArray[Any]: ------- NumPy array of this buffer (might be a data copy) """ - return np.asanyarray(self._data) + ... @property def dtype(self) -> np.dtype[Any]: @@ -412,13 +441,11 @@ def squeeze(self, axis: tuple[int, ...]) -> Self: def astype(self, dtype: npt.DTypeLike, order: Literal["K", "A", "C", "F"] = "K") -> Self: return self.__class__(self._data.astype(dtype=dtype, order=order)) - def __getitem__(self, key: Any) -> Self: - return self.__class__(np.asanyarray(self._data.__getitem__(key))) + @abstractmethod + def __getitem__(self, key: Any) -> Self: ... - def __setitem__(self, key: Any, value: Any) -> None: - if isinstance(value, NDBuffer): - value = value._data - self._data.__setitem__(key, value) + @abstractmethod + def __setitem__(self, key: Any, value: Any) -> None: ... def __len__(self) -> int: return self._data.__len__() @@ -439,34 +466,6 @@ def transpose(self, axes: SupportsIndex | Sequence[SupportsIndex] | None) -> Sel return self.__class__(self._data.transpose(axes)) -def as_numpy_array_wrapper( - func: Callable[[npt.NDArray[Any]], bytes], buf: Buffer, prototype: BufferPrototype -) -> Buffer: - """Converts the input of `func` to a numpy array and the output back to `Buffer`. - - This function is useful when calling a `func` that only support host memory such - as `GZip.decode` and `Blosc.decode`. In this case, use this wrapper to convert - the input `buf` to a Numpy array and convert the result back into a `Buffer`. - - Parameters - ---------- - func - The callable that will be called with the converted `buf` as input. - `func` must return bytes, which will be converted into a `Buffer` - before returned. - buf - The buffer that will be converted to a Numpy array before given as - input to `func`. - prototype - The prototype of the output buffer. - - Returns - ------- - The result of `func` converted to a `Buffer` - """ - return prototype.buffer.from_bytes(func(buf.as_numpy_array())) - - class BufferPrototype(NamedTuple): """Prototype of the Buffer and NDBuffer class @@ -487,12 +486,3 @@ class BufferPrototype(NamedTuple): # The default buffer prototype used throughout the Zarr codebase. def default_buffer_prototype() -> BufferPrototype: return BufferPrototype(buffer=get_buffer_class(), nd_buffer=get_ndbuffer_class()) - - -# The numpy prototype used for E.g. when reading the shard index -def numpy_buffer_prototype() -> BufferPrototype: - return BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) - - -register_buffer(Buffer) -register_ndbuffer(NDBuffer) diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py new file mode 100644 index 0000000000..0953805223 --- /dev/null +++ b/src/zarr/core/buffer/cpu.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +from collections.abc import Callable, Iterable +from typing import ( + TYPE_CHECKING, + Any, + Literal, +) + +import numpy as np +import numpy.typing as npt + +from zarr.core.buffer import core +from zarr.core.buffer.core import ArrayLike, NDArrayLike +from zarr.registry import ( + register_buffer, + register_ndbuffer, +) + +if TYPE_CHECKING: + from typing_extensions import Self + + from zarr.core.common import BytesLike + + +class Buffer(core.Buffer): + """A flat contiguous memory block + + We use Buffer throughout Zarr to represent a contiguous block of memory. + + A Buffer is backed by a underlying array-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + array-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + This buffer is untyped, so all indexing and sizes are in bytes. + + Parameters + ---------- + array_like + array-like object that must be 1-dim, contiguous, and byte dtype. + """ + + def __init__(self, array_like: ArrayLike): + super().__init__(array_like) + + @classmethod + def create_zero_length(cls) -> Self: + return cls(np.array([], dtype="b")) + + @classmethod + def from_buffer(cls, buffer: core.Buffer) -> Self: + """Create a new buffer of an existing Buffer + + This is useful if you want to ensure that an existing buffer is + of the correct subclass of Buffer. E.g., MemoryStore uses this + to return a buffer instance of the subclass specified by its + BufferPrototype argument. + + Typically, this only copies data if the data has to be moved between + memory types, such as from host to device memory. + + Parameters + ---------- + buffer + buffer object. + + Returns + ------- + A new buffer representing the content of the input buffer + + Note + ---- + Subclasses of `Buffer` must override this method to implement + more optimal conversions that avoid copies where possible + """ + return cls.from_array_like(buffer.as_numpy_array()) + + @classmethod + def from_bytes(cls, bytes_like: BytesLike) -> Self: + """Create a new buffer of a bytes-like object (host memory) + + Parameters + ---------- + bytes_like + bytes-like object + + Returns + ------- + New buffer representing `bytes_like` + """ + return cls.from_array_like(np.frombuffer(bytes_like, dtype="b")) + + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Notes + ----- + Might have to copy data, consider using `.as_array_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + return np.asanyarray(self._data) + + def __add__(self, other: core.Buffer) -> Self: + """Concatenate two buffers""" + + other_array = other.as_array_like() + assert other_array.dtype == np.dtype("b") + return self.__class__( + np.concatenate((np.asanyarray(self._data), np.asanyarray(other_array))) + ) + + +class NDBuffer(core.NDBuffer): + """An n-dimensional memory block + + We use NDBuffer throughout Zarr to represent a n-dimensional memory block. + + A NDBuffer is backed by a underlying ndarray-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + ndarray-like instance can be copied/converted to a regular Numpy array + (host memory). + + Notes + ----- + The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer + is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + in order to use Python's type system to differentiate between the contiguous + Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the + two classes separate. + + Parameters + ---------- + ndarray_like + ndarray-like object that is convertible to a regular Numpy array. + """ + + def __init__(self, array: NDArrayLike): + super().__init__(array) + + @classmethod + def create( + cls, + *, + shape: Iterable[int], + dtype: npt.DTypeLike, + order: Literal["C", "F"] = "C", + fill_value: Any | None = None, + ) -> Self: + ret = cls(np.empty(shape=tuple(shape), dtype=dtype, order=order)) + if fill_value is not None: + ret.fill(fill_value) + return ret + + @classmethod + def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: + return cls.from_ndarray_like(np.asanyarray(array_like)) + + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Warnings + -------- + Might have to copy data, consider using `.as_ndarray_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + return np.asanyarray(self._data) + + def __getitem__(self, key: Any) -> Self: + return self.__class__(np.asanyarray(self._data.__getitem__(key))) + + def __setitem__(self, key: Any, value: Any) -> None: + if isinstance(value, NDBuffer): + value = value._data + self._data.__setitem__(key, value) + + +def as_numpy_array_wrapper( + func: Callable[[npt.NDArray[Any]], bytes], buf: core.Buffer, prototype: core.BufferPrototype +) -> core.Buffer: + """Converts the input of `func` to a numpy array and the output back to `Buffer`. + + This function is useful when calling a `func` that only support host memory such + as `GZip.decode` and `Blosc.decode`. In this case, use this wrapper to convert + the input `buf` to a Numpy array and convert the result back into a `Buffer`. + + Parameters + ---------- + func + The callable that will be called with the converted `buf` as input. + `func` must return bytes, which will be converted into a `Buffer` + before returned. + buf + The buffer that will be converted to a Numpy array before given as + input to `func`. + prototype + The prototype of the output buffer. + + Returns + ------- + The result of `func` converted to a `Buffer` + """ + return prototype.buffer.from_bytes(func(buf.as_numpy_array())) + + +# CPU buffer prototype using numpy arrays +buffer_prototype = core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) +# default_buffer_prototype = buffer_prototype + + +# The numpy prototype used for E.g. when reading the shard index +def numpy_buffer_prototype() -> core.BufferPrototype: + return core.BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) + + +register_buffer(Buffer) +register_ndbuffer(NDBuffer) diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py new file mode 100644 index 0000000000..9d38df2abf --- /dev/null +++ b/src/zarr/core/buffer/gpu.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import warnings +from collections.abc import Iterable +from typing import ( + TYPE_CHECKING, + Any, + Literal, + cast, +) + +import numpy as np +import numpy.typing as npt + +from zarr.core.buffer import core +from zarr.core.buffer.core import ArrayLike, BufferPrototype, NDArrayLike + +if TYPE_CHECKING: + from typing_extensions import Self + + from zarr.core.common import BytesLike + +try: + import cupy as cp +except ImportError: + cp = None + + +class Buffer(core.Buffer): + """A flat contiguous memory block on the GPU + + We use Buffer throughout Zarr to represent a contiguous block of memory. + + A Buffer is backed by a underlying array-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + array-like instance can be copied/converted to a regular Numpy array + (host memory). + + Note + ---- + This buffer is untyped, so all indexing and sizes are in bytes. + + Parameters + ---------- + array_like + array-like object that must be 1-dim, contiguous, and byte dtype. + """ + + def __init__(self, array_like: ArrayLike): + if cp is None: + raise ImportError( + "Cannot use zarr.buffer.gpu.Buffer without cupy. Please install cupy." + ) + + if array_like.ndim != 1: + raise ValueError("array_like: only 1-dim allowed") + if array_like.dtype != np.dtype("b"): + raise ValueError("array_like: only byte dtype allowed") + + if not hasattr(array_like, "__cuda_array_interface__"): + # Slow copy based path for arrays that don't support the __cuda_array_interface__ + # TODO: Add a fast zero-copy path for arrays that support the dlpack protocol + msg = ( + "Creating a zarr.buffer.gpu.Buffer with an array that does not support the " + "__cuda_array_interface__ for zero-copy transfers, " + "falling back to slow copy based path" + ) + warnings.warn( + msg, + stacklevel=2, + ) + self._data = cp.asarray(array_like) + + @classmethod + def create_zero_length(cls) -> Self: + """Create an empty buffer with length zero + + Returns + ------- + New empty 0-length buffer + """ + return cls(cp.array([], dtype="b")) + + @classmethod + def from_buffer(cls, buffer: core.Buffer) -> Self: + """Create an GPU Buffer given an arbitrary Buffer + This will try to be zero-copy if `buffer` is already on the + GPU and will trigger a copy if not. + + Returns + ------- + New GPU Buffer constructed from `buffer` + """ + return cls(buffer.as_array_like()) + + @classmethod + def from_bytes(cls, bytes_like: BytesLike) -> Self: + return cls.from_array_like(cp.frombuffer(bytes_like, dtype="b")) + + def as_numpy_array(self) -> npt.NDArray[Any]: + return cast(npt.NDArray[Any], cp.asnumpy(self._data)) + + def __add__(self, other: core.Buffer) -> Self: + other_array = other.as_array_like() + assert other_array.dtype == np.dtype("b") + gpu_other = Buffer(other_array) + gpu_other_array = gpu_other.as_array_like() + return self.__class__( + cp.concatenate((cp.asanyarray(self._data), cp.asanyarray(gpu_other_array))) + ) + + +class NDBuffer(core.NDBuffer): + """A n-dimensional memory block on the GPU + + We use NDBuffer throughout Zarr to represent a n-dimensional memory block. + + A NDBuffer is backed by a underlying ndarray-like instance that represents + the memory. The memory type is unspecified; can be regular host memory, + CUDA device memory, or something else. The only requirement is that the + ndarray-like instance can be copied/converted to a regular Numpy array + (host memory). + + Note + ---- + The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer + is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, + in order to use Python's type system to differentiate between the contiguous + Buffer and the n-dim (non-contiguous) NDBuffer, we keep the definition of the + two classes separate. + + Parameters + ---------- + ndarray_like + ndarray-like object that is convertible to a regular Numpy array. + """ + + def __init__(self, array: NDArrayLike): + if cp is None: + raise ImportError( + "Cannot use zarr.buffer.gpu.NDBuffer without cupy. Please install cupy." + ) + + # assert array.ndim > 0 + assert array.dtype != object + self._data = array + + if not hasattr(array, "__cuda_array_interface__"): + # Slow copy based path for arrays that don't support the __cuda_array_interface__ + # TODO: Add a fast zero-copy path for arrays that support the dlpack protocol + msg = ( + "Creating a zarr.buffer.gpu.NDBuffer with an array that does not support the " + "__cuda_array_interface__ for zero-copy transfers, " + "falling back to slow copy based path" + ) + warnings.warn( + msg, + stacklevel=2, + ) + self._data = cp.asarray(array) + + @classmethod + def create( + cls, + *, + shape: Iterable[int], + dtype: npt.DTypeLike, + order: Literal["C", "F"] = "C", + fill_value: Any | None = None, + ) -> Self: + ret = cls(cp.empty(shape=tuple(shape), dtype=dtype, order=order)) + if fill_value is not None: + ret.fill(fill_value) + return ret + + @classmethod + def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: + """Create a new buffer of Numpy array-like object + + Parameters + ---------- + array_like + Object that can be coerced into a Numpy array + + Returns + ------- + New buffer representing `array_like` + """ + return cls(cp.asarray(array_like)) + + def as_numpy_array(self) -> npt.NDArray[Any]: + """Returns the buffer as a NumPy array (host memory). + + Warning + ------- + Might have to copy data, consider using `.as_ndarray_like()` instead. + + Returns + ------- + NumPy array of this buffer (might be a data copy) + """ + return cast(npt.NDArray[Any], cp.asnumpy(self._data)) + + def __getitem__(self, key: Any) -> Self: + return self.__class__(self._data.__getitem__(key)) + + def __setitem__(self, key: Any, value: Any) -> None: + if isinstance(value, NDBuffer): + value = value._data + elif isinstance(value, core.NDBuffer): + gpu_value = NDBuffer(value.as_ndarray_like()) + value = gpu_value._data + self._data.__setitem__(key, value) + + +buffer_prototype = BufferPrototype(buffer=Buffer, nd_buffer=NDBuffer) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 67f5c74347..611d1faea5 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -60,8 +60,8 @@ def reset(self) -> None: "sharding_indexed": "zarr.codecs.sharding.ShardingCodec", "transpose": "zarr.codecs.transpose.TransposeCodec", }, - "buffer": "zarr.core.buffer.Buffer", - "ndbuffer": "zarr.core.buffer.NDBuffer", + "buffer": "zarr.core.buffer.cpu.Buffer", + "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", } ], ) diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 117fd69ec0..c817b0963a 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING from zarr.abc.store import Store -from zarr.core.buffer import Buffer +from zarr.core.buffer import Buffer, gpu from zarr.core.common import concurrent_map from zarr.store._utils import _normalize_interval_index @@ -126,3 +126,39 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: for key in keys_unique: yield key + + +class GpuMemoryStore(MemoryStore): + """A GPU only memory store that stores every chunk in GPU memory irrespective + of the original location. This guarantees that chunks will always be in GPU + memory for downstream processing. For location agnostic use cases, it would + be better to use `MemoryStore` instead. + """ + + _store_dict: MutableMapping[str, Buffer] + + def __init__( + self, + store_dict: MutableMapping[str, Buffer] | None = None, + *, + mode: AccessModeLiteral = "r", + ): + super().__init__(mode=mode) + if store_dict: + self._store_dict = {k: gpu.Buffer.from_buffer(store_dict[k]) for k in iter(store_dict)} + + def __str__(self) -> str: + return f"gpumemory://{id(self._store_dict)}" + + def __repr__(self) -> str: + return f"GpuMemoryStore({str(self)!r})" + + async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: + self._check_writable() + assert isinstance(key, str) + if not isinstance(value, Buffer): + raise TypeError(f"Expected Buffer. Got {type(value)}.") + + # Convert to gpu.Buffer + gpu_value = value if isinstance(value, gpu.Buffer) else gpu.Buffer.from_buffer(value) + await super().set(key, gpu_value, byte_range=byte_range) diff --git a/src/zarr/testing/buffer.py b/src/zarr/testing/buffer.py index ee170a5dd3..a6120ef2f9 100644 --- a/src/zarr/testing/buffer.py +++ b/src/zarr/testing/buffer.py @@ -7,7 +7,7 @@ import numpy as np import numpy.typing as npt -from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer +from zarr.core.buffer import Buffer, BufferPrototype, cpu from zarr.store import MemoryStore if TYPE_CHECKING: @@ -25,11 +25,11 @@ class TestNDArrayLike(np.ndarray): """An example of a ndarray-like class""" -class TestBuffer(Buffer): +class TestBuffer(cpu.Buffer): """Example of a custom Buffer that handles ArrayLike""" -class NDBufferUsingTestNDArrayLike(NDBuffer): +class NDBufferUsingTestNDArrayLike(cpu.NDBuffer): """Example of a custom NDBuffer that handles MyNDArrayLike""" @classmethod diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 11978b4121..925087ae67 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -11,10 +11,12 @@ S = TypeVar("S", bound=Store) +B = TypeVar("B", bound=Buffer) -class StoreTests(Generic[S]): +class StoreTests(Generic[S, B]): store_cls: type[S] + buffer_cls: type[B] def set(self, store: S, key: str, value: Buffer) -> None: """ @@ -60,7 +62,7 @@ async def test_not_writable_store_raises(self, store_kwargs: dict[str, Any]) -> # set with pytest.raises(ValueError): - await store.set("foo", Buffer.from_bytes(b"bar")) + await store.set("foo", self.buffer_cls.from_bytes(b"bar")) # delete with pytest.raises(ValueError): @@ -87,7 +89,7 @@ async def test_get( """ Ensure that data can be read from the store using the store.get method. """ - data_buf = Buffer.from_bytes(data) + data_buf = self.buffer_cls.from_bytes(data) self.set(store, key, data_buf) observed = await store.get(key, prototype=default_buffer_prototype(), byte_range=byte_range) start, length = _normalize_interval_index(data_buf, interval=byte_range) @@ -101,7 +103,7 @@ async def test_set(self, store: S, key: str, data: bytes) -> None: Ensure that data can be written to the store using the store.set method. """ assert not store.mode.readonly - data_buf = Buffer.from_bytes(data) + data_buf = self.buffer_cls.from_bytes(data) await store.set(key, data_buf) observed = self.get(store, key) assert_bytes_equal(observed, data_buf) @@ -120,7 +122,7 @@ async def test_get_partial_values( ) -> None: # put all of the data for key, _ in key_ranges: - self.set(store, key, Buffer.from_bytes(bytes(key, encoding="utf-8"))) + self.set(store, key, self.buffer_cls.from_bytes(bytes(key, encoding="utf-8"))) # read back just part of it observed_maybe = await store.get_partial_values( @@ -148,28 +150,28 @@ async def test_get_partial_values( async def test_exists(self, store: S) -> None: assert not await store.exists("foo") - await store.set("foo/zarr.json", Buffer.from_bytes(b"bar")) + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) assert await store.exists("foo/zarr.json") async def test_delete(self, store: S) -> None: - await store.set("foo/zarr.json", Buffer.from_bytes(b"bar")) + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) assert await store.exists("foo/zarr.json") await store.delete("foo/zarr.json") assert not await store.exists("foo/zarr.json") async def test_empty(self, store: S) -> None: assert await store.empty() - self.set(store, "key", Buffer.from_bytes(bytes("something", encoding="utf-8"))) + self.set(store, "key", self.buffer_cls.from_bytes(bytes("something", encoding="utf-8"))) assert not await store.empty() async def test_clear(self, store: S) -> None: - self.set(store, "key", Buffer.from_bytes(bytes("something", encoding="utf-8"))) + self.set(store, "key", self.buffer_cls.from_bytes(bytes("something", encoding="utf-8"))) await store.clear() assert await store.empty() async def test_list(self, store: S) -> None: assert [k async for k in store.list()] == [] - await store.set("foo/zarr.json", Buffer.from_bytes(b"bar")) + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) keys = [k async for k in store.list()] assert keys == ["foo/zarr.json"], keys @@ -178,7 +180,7 @@ async def test_list(self, store: S) -> None: key = f"foo/c/{i}" expected.append(key) await store.set( - f"foo/c/{i}", Buffer.from_bytes(i.to_bytes(length=3, byteorder="little")) + f"foo/c/{i}", self.buffer_cls.from_bytes(i.to_bytes(length=3, byteorder="little")) ) @pytest.mark.xfail @@ -190,12 +192,12 @@ async def test_list_dir(self, store: S) -> None: out = [k async for k in store.list_dir("")] assert out == [] assert [k async for k in store.list_dir("foo")] == [] - await store.set("foo/zarr.json", Buffer.from_bytes(b"bar")) - await store.set("group-0/zarr.json", Buffer.from_bytes(b"\x01")) # group - await store.set("group-0/group-1/zarr.json", Buffer.from_bytes(b"\x01")) # group - await store.set("group-0/group-1/a1/zarr.json", Buffer.from_bytes(b"\x01")) - await store.set("group-0/group-1/a2/zarr.json", Buffer.from_bytes(b"\x01")) - await store.set("group-0/group-1/a3/zarr.json", Buffer.from_bytes(b"\x01")) + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) + await store.set("group-0/zarr.json", self.buffer_cls.from_bytes(b"\x01")) # group + await store.set("group-0/group-1/zarr.json", self.buffer_cls.from_bytes(b"\x01")) # group + await store.set("group-0/group-1/a1/zarr.json", self.buffer_cls.from_bytes(b"\x01")) + await store.set("group-0/group-1/a2/zarr.json", self.buffer_cls.from_bytes(b"\x01")) + await store.set("group-0/group-1/a3/zarr.json", self.buffer_cls.from_bytes(b"\x01")) keys_expected = ["foo", "group-0"] keys_observed = [k async for k in store.list_dir("")] diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index 3a70f96d44..a21a0be708 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,5 +1,9 @@ from __future__ import annotations +from typing import Any, cast + +import pytest + from zarr.core.buffer import Buffer from zarr.core.common import BytesLike @@ -18,3 +22,21 @@ def assert_bytes_equal(b1: Buffer | BytesLike | None, b2: Buffer | BytesLike | N if isinstance(b2, Buffer): b2 = b2.to_bytes() assert b1 == b2 + + +def has_cupy() -> bool: + try: + import cupy + + return cast(bool, cupy.cuda.runtime.getDeviceCount() > 0) + except ImportError: + return False + except cupy.cuda.runtime.CUDARuntimeError: + return False + + +# Decorator for GPU tests +def gpu_test(func: Any) -> Any: + return pytest.mark.gpu( + pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")(func) + ) diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index d8af484ee6..c67d7bb4fd 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -94,6 +94,9 @@ async def async_group(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> As def xp(request: pytest.FixtureRequest) -> Iterator[ModuleType]: """Fixture to parametrize over numpy-like libraries""" + if request.param == "cupy": + request.node.add_marker(pytest.mark.gpu) + yield pytest.importorskip(request.param) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 298a7f7eed..419f01c720 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -10,14 +10,21 @@ from zarr.codecs.gzip import GzipCodec from zarr.codecs.transpose import TransposeCodec from zarr.codecs.zstd import ZstdCodec -from zarr.core.buffer import ArrayLike, BufferPrototype, NDArrayLike, numpy_buffer_prototype +from zarr.core.buffer import ArrayLike, BufferPrototype, NDArrayLike, cpu, gpu from zarr.store.common import StorePath +from zarr.store.memory import MemoryStore from zarr.testing.buffer import ( NDBufferUsingTestNDArrayLike, StoreExpectingTestBuffer, TestBuffer, TestNDArrayLike, ) +from zarr.testing.utils import gpu_test + +try: + import cupy as cp +except ImportError: + cp = None def test_nd_array_like(xp): @@ -52,6 +59,31 @@ async def test_async_array_prototype(): assert np.array_equal(expect, got) +@gpu_test +@pytest.mark.asyncio +async def test_async_array_gpu_prototype(): + """Test the use of the GPU buffer prototype""" + + expect = cp.zeros((9, 9), dtype="uint16", order="F") + a = await AsyncArray.create( + StorePath(MemoryStore(mode="w")) / "test_async_array_gpu_prototype", + shape=expect.shape, + chunk_shape=(5, 5), + dtype=expect.dtype, + fill_value=0, + ) + expect[1:4, 3:6] = cp.ones((3, 3)) + + await a.setitem( + selection=(slice(1, 4), slice(3, 6)), + value=cp.ones((3, 3)), + prototype=gpu.buffer_prototype, + ) + got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=gpu.buffer_prototype) + assert isinstance(got, cp.ndarray) + assert cp.array_equal(expect, got) + + @pytest.mark.asyncio async def test_codecs_use_of_prototype(): expect = np.zeros((10, 10), dtype="uint16", order="F") @@ -84,8 +116,39 @@ async def test_codecs_use_of_prototype(): assert np.array_equal(expect, got) +@gpu_test +@pytest.mark.asyncio +async def test_codecs_use_of_gpu_prototype(): + expect = cp.zeros((10, 10), dtype="uint16", order="F") + a = await AsyncArray.create( + StorePath(MemoryStore(mode="w")) / "test_codecs_use_of_gpu_prototype", + shape=expect.shape, + chunk_shape=(5, 5), + dtype=expect.dtype, + fill_value=0, + codecs=[ + TransposeCodec(order=(1, 0)), + BytesCodec(), + BloscCodec(), + Crc32cCodec(), + GzipCodec(), + ZstdCodec(), + ], + ) + expect[:] = cp.arange(100).reshape(10, 10) + + await a.setitem( + selection=(slice(0, 10), slice(0, 10)), + value=expect[:], + prototype=gpu.buffer_prototype, + ) + got = await a.getitem(selection=(slice(0, 10), slice(0, 10)), prototype=gpu.buffer_prototype) + assert isinstance(got, cp.ndarray) + assert cp.array_equal(expect, got) + + def test_numpy_buffer_prototype(): - buffer = numpy_buffer_prototype().buffer.create_zero_length() - ndbuffer = numpy_buffer_prototype().nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) + buffer = cpu.buffer_prototype.buffer.create_zero_length() + ndbuffer = cpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) assert isinstance(buffer.as_array_like(), np.ndarray) assert isinstance(ndbuffer.as_ndarray_like(), np.ndarray) diff --git a/tests/v3/test_config.py b/tests/v3/test_config.py index 881833797c..b3e04e51da 100644 --- a/tests/v3/test_config.py +++ b/tests/v3/test_config.py @@ -46,8 +46,8 @@ def test_config_defaults_set() -> None: "path": "zarr.codecs.pipeline.BatchedCodecPipeline", "batch_size": 1, }, - "buffer": "zarr.core.buffer.Buffer", - "ndbuffer": "zarr.core.buffer.NDBuffer", + "buffer": "zarr.core.buffer.cpu.Buffer", + "ndbuffer": "zarr.core.buffer.cpu.NDBuffer", "codecs": { "blosc": "zarr.codecs.blosc.BloscCodec", "gzip": "zarr.codecs.gzip.GzipCodec", diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index eb7b1f30dd..7592c77201 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -7,7 +7,7 @@ from _pytest.compat import LEGACY_PATH from zarr import Array, AsyncArray, AsyncGroup, Group -from zarr.core.buffer import Buffer +from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ZarrFormat from zarr.core.group import GroupMetadata from zarr.core.sync import sync @@ -97,11 +97,18 @@ def test_group_members(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) # add an extra object to the domain of the group. # the list of children should ignore this object. - sync(store.set(f"{path}/extra_object-1", Buffer.from_bytes(b"000000"))) + sync( + store.set(f"{path}/extra_object-1", default_buffer_prototype().buffer.from_bytes(b"000000")) + ) # add an extra object under a directory-like prefix in the domain of the group. # this creates a directory with a random key in it # this should not show up as a member - sync(store.set(f"{path}/extra_directory/extra_object-2", Buffer.from_bytes(b"000000"))) + sync( + store.set( + f"{path}/extra_directory/extra_object-2", + default_buffer_prototype().buffer.from_bytes(b"000000"), + ) + ) members_observed = group.members() # members are not guaranteed to be ordered, so sort before comparing assert sorted(dict(members_observed)) == sorted(members_expected) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 7895151b5f..5805de3035 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -12,7 +12,7 @@ import zarr from zarr.abc.store import Store -from zarr.core.buffer import BufferPrototype, NDBuffer +from zarr.core.buffer import BufferPrototype, default_buffer_prototype from zarr.core.common import ChunkCoords from zarr.core.indexing import ( make_slice_selection, @@ -136,7 +136,7 @@ def test_get_basic_selection_0d(store: StorePath, use_out: bool, value: Any, dty if use_out: # test out param - b = NDBuffer.from_numpy_array(np.zeros_like(arr_np)) + b = default_buffer_prototype().nd_buffer.from_numpy_array(np.zeros_like(arr_np)) arr_z.get_basic_selection(Ellipsis, out=b) assert_array_equal(arr_np, b.as_ndarray_like()) @@ -247,7 +247,9 @@ def _test_get_basic_selection(a, z, selection): assert_array_equal(expect, actual) # test out param - b = NDBuffer.from_numpy_array(np.empty(shape=expect.shape, dtype=expect.dtype)) + b = default_buffer_prototype().nd_buffer.from_numpy_array( + np.empty(shape=expect.shape, dtype=expect.dtype) + ) z.get_basic_selection(selection, out=b) assert_array_equal(expect, b.as_numpy_array()) diff --git a/tests/v3/test_store/test_local.py b/tests/v3/test_store/test_local.py index 8afa4fef3a..59cae22de3 100644 --- a/tests/v3/test_store/test_local.py +++ b/tests/v3/test_store/test_local.py @@ -2,16 +2,17 @@ import pytest -from zarr.core.buffer import Buffer +from zarr.core.buffer import Buffer, cpu from zarr.store.local import LocalStore from zarr.testing.store import StoreTests -class TestLocalStore(StoreTests[LocalStore]): +class TestLocalStore(StoreTests[LocalStore, cpu.Buffer]): store_cls = LocalStore + buffer_cls = cpu.Buffer def get(self, store: LocalStore, key: str) -> Buffer: - return Buffer.from_bytes((store.root / key).read_bytes()) + return self.buffer_cls.from_bytes((store.root / key).read_bytes()) def set(self, store: LocalStore, key: str, value: Buffer) -> None: parent = (store.root / key).parent diff --git a/tests/v3/test_store/test_memory.py b/tests/v3/test_store/test_memory.py index 51ecf46709..f76423c631 100644 --- a/tests/v3/test_store/test_memory.py +++ b/tests/v3/test_store/test_memory.py @@ -2,13 +2,15 @@ import pytest -from zarr.core.buffer import Buffer -from zarr.store.memory import MemoryStore +from zarr.core.buffer import Buffer, cpu, gpu +from zarr.store.memory import GpuMemoryStore, MemoryStore from zarr.testing.store import StoreTests +from zarr.testing.utils import gpu_test -class TestMemoryStore(StoreTests[MemoryStore]): +class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]): store_cls = MemoryStore + buffer_cls = cpu.Buffer def set(self, store: MemoryStore, key: str, value: Buffer) -> None: store._store_dict[key] = value @@ -40,3 +42,38 @@ def test_store_supports_partial_writes(self, store: MemoryStore) -> None: def test_list_prefix(self, store: MemoryStore) -> None: assert True + + +@gpu_test +class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): + store_cls = GpuMemoryStore + buffer_cls = gpu.Buffer + + def set(self, store: GpuMemoryStore, key: str, value: Buffer) -> None: + store._store_dict[key] = value + + def get(self, store: MemoryStore, key: str) -> Buffer: + return store._store_dict[key] + + @pytest.fixture(scope="function", params=[None, {}]) + def store_kwargs(self, request) -> dict[str, str | None | dict[str, Buffer]]: + return {"store_dict": request.param, "mode": "r+"} + + @pytest.fixture(scope="function") + def store(self, store_kwargs: str | None | dict[str, gpu.Buffer]) -> GpuMemoryStore: + return self.store_cls(**store_kwargs) + + def test_store_repr(self, store: GpuMemoryStore) -> None: + assert str(store) == f"gpumemory://{id(store._store_dict)}" + + def test_store_supports_writes(self, store: GpuMemoryStore) -> None: + assert store.supports_writes + + def test_store_supports_listing(self, store: GpuMemoryStore) -> None: + assert store.supports_listing + + def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: + assert store.supports_partial_writes + + def test_list_prefix(self, store: GpuMemoryStore) -> None: + assert True diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index 2eec9d29cf..b09bf24e24 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -4,7 +4,7 @@ import pytest from upath import UPath -from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.sync import sync from zarr.store import RemoteStore from zarr.testing.store import StoreTests @@ -88,7 +88,7 @@ async def test_basic(): assert not await alist(store.list()) assert not await store.exists("foo") data = b"hello" - await store.set("foo", Buffer.from_bytes(data)) + await store.set("foo", cpu.Buffer.from_bytes(data)) assert await store.exists("foo") assert (await store.get("foo", prototype=default_buffer_prototype())).to_bytes() == data out = await store.get_partial_values( @@ -97,8 +97,9 @@ async def test_basic(): assert out[0].to_bytes() == data[1:] -class TestRemoteStoreS3(StoreTests[RemoteStore]): +class TestRemoteStoreS3(StoreTests[RemoteStore, cpu.Buffer]): store_cls = RemoteStore + buffer_cls = cpu.Buffer @pytest.fixture(scope="function", params=("use_upath", "use_str")) def store_kwargs(self, request) -> dict[str, str | bool]: @@ -131,7 +132,7 @@ def get(self, store: RemoteStore, key: str) -> Buffer: anon=store._fs.anon, endpoint_url=store._fs.endpoint_url, ) - return Buffer.from_bytes(fs.cat(f"{store.path}/{key}")) + return self.buffer_cls.from_bytes(fs.cat(f"{store.path}/{key}")) def set(self, store: RemoteStore, key: str, value: Buffer) -> None: # make a new, synchronous instance of the filesystem because this test is run in sync code diff --git a/tests/v3/test_store/test_stateful_store.py b/tests/v3/test_store/test_stateful_store.py index 68bd11bbe8..a8f51e96e6 100644 --- a/tests/v3/test_store/test_stateful_store.py +++ b/tests/v3/test_store/test_stateful_store.py @@ -12,7 +12,7 @@ import zarr from zarr.abc.store import AccessMode, Store -from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype +from zarr.core.buffer import BufferPrototype, cpu, default_buffer_prototype from zarr.store import MemoryStore from zarr.testing.strategies import key_ranges, paths @@ -112,7 +112,7 @@ def __init__(self): def set(self, key: str, data: bytes) -> None: note(f"(set) Setting {key!r} with {data}") assert not self.store.mode.readonly - data_buf = Buffer.from_bytes(data) + data_buf = cpu.Buffer.from_bytes(data) self.store.set(key, data_buf) self.model[key] = data_buf From 0b5483a7958e2ae5512a14eb424a84b2a75dd727 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 30 Aug 2024 11:26:35 -0700 Subject: [PATCH 0130/1043] [v3] clean up `create_array` signatures in group/asyncgroup classes (#2132) * clean up create_array signatures in group/asyncgroup classes * fix members test --- src/zarr/core/group.py | 16 +++++++++------- tests/v3/test_group.py | 34 +++++++++++++++++----------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 2c26cac3b1..566acf7ab8 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -311,13 +311,14 @@ def info(self) -> None: async def create_group( self, - path: str, + name: str, + *, exists_ok: bool = False, attributes: dict[str, Any] | None = None, ) -> AsyncGroup: attributes = attributes or {} return await type(self).create( - self.store_path / path, + self.store_path / name, attributes=attributes, exists_ok=exists_ok, zarr_format=self.metadata.zarr_format, @@ -325,7 +326,8 @@ async def create_group( async def create_array( self, - path: str, + name: str, + *, shape: ChunkCoords, dtype: npt.DTypeLike = "float64", fill_value: Any | None = None, @@ -356,7 +358,7 @@ async def create_array( Parameters ---------- - path: str + name: str The name of the array. shape: tuple[int, ...] The shape of the array. @@ -392,7 +394,7 @@ async def create_array( """ return await AsyncArray.create( - self.store_path / path, + self.store_path / name, shape=shape, dtype=dtype, chunk_shape=chunk_shape, @@ -789,7 +791,7 @@ def create_array( return Array( self._sync( self._async_group.create_array( - path=name, + name=name, shape=shape, dtype=dtype, fill_value=fill_value, @@ -912,7 +914,7 @@ def array( return Array( self._sync( self._async_group.create_array( - path=name, + name=name, shape=shape, dtype=dtype, fill_value=fill_value, diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 7592c77201..fa0df3673e 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -564,11 +564,11 @@ async def test_asyncgroup_getitem(store: LocalStore | MemoryStore, zarr_format: """ agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) - sub_array_path = "sub_array" + array_name = "sub_array" sub_array = await agroup.create_array( - path=sub_array_path, shape=(10,), dtype="uint8", chunk_shape=(2,) + name=array_name, shape=(10,), dtype="uint8", chunk_shape=(2,) ) - assert await agroup.getitem(sub_array_path) == sub_array + assert await agroup.getitem(array_name) == sub_array sub_group_path = "sub_group" sub_group = await agroup.create_group(sub_group_path, attributes={"foo": 100}) @@ -581,18 +581,18 @@ async def test_asyncgroup_getitem(store: LocalStore | MemoryStore, zarr_format: async def test_asyncgroup_delitem(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) - sub_array_path = "sub_array" + array_name = "sub_array" _ = await agroup.create_array( - path=sub_array_path, shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} + name=array_name, shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} ) - await agroup.delitem(sub_array_path) + await agroup.delitem(array_name) # todo: clean up the code duplication here if zarr_format == 2: - assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zarray") - assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zattrs") + assert not await agroup.store_path.store.exists(array_name + "/" + ".zarray") + assert not await agroup.store_path.store.exists(array_name + "/" + ".zattrs") elif zarr_format == 3: - assert not await agroup.store_path.store.exists(sub_array_path + "/" + "zarr.json") + assert not await agroup.store_path.store.exists(array_name + "/" + "zarr.json") else: raise AssertionError @@ -600,10 +600,10 @@ async def test_asyncgroup_delitem(store: LocalStore | MemoryStore, zarr_format: _ = await agroup.create_group(sub_group_path, attributes={"foo": 100}) await agroup.delitem(sub_group_path) if zarr_format == 2: - assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zgroup") - assert not await agroup.store_path.store.exists(sub_array_path + "/" + ".zattrs") + assert not await agroup.store_path.store.exists(array_name + "/" + ".zgroup") + assert not await agroup.store_path.store.exists(array_name + "/" + ".zattrs") elif zarr_format == 3: - assert not await agroup.store_path.store.exists(sub_array_path + "/" + "zarr.json") + assert not await agroup.store_path.store.exists(array_name + "/" + "zarr.json") else: raise AssertionError @@ -615,7 +615,7 @@ async def test_asyncgroup_create_group( agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) sub_node_path = "sub_group" attributes = {"foo": 999} - subnode = await agroup.create_group(path=sub_node_path, attributes=attributes) + subnode = await agroup.create_group(name=sub_node_path, attributes=attributes) assert isinstance(subnode, AsyncGroup) assert subnode.attrs == attributes @@ -645,7 +645,7 @@ async def test_asyncgroup_create_array( sub_node_path = "sub_array" subnode = await agroup.create_array( - path=sub_node_path, + name=sub_node_path, shape=shape, dtype=dtype, chunk_shape=chunk_shape, @@ -684,11 +684,11 @@ async def test_group_members_async(store: LocalStore | MemoryStore): GroupMetadata(), store_path=StorePath(store=store, path="root"), ) - a0 = await group.create_array("a0", (1,)) + a0 = await group.create_array("a0", shape=(1,)) g0 = await group.create_group("g0") - a1 = await g0.create_array("a1", (1,)) + a1 = await g0.create_array("a1", shape=(1,)) g1 = await g0.create_group("g1") - a2 = await g1.create_array("a2", (1,)) + a2 = await g1.create_array("a2", shape=(1,)) g2 = await g1.create_group("g2") # immediate children From 6656d1a2d8f95dfb29724327d9dcff69667dc9d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 11:41:00 +0100 Subject: [PATCH 0131/1043] Bump the actions group with 2 updates (#2146) Bumps the actions group with 2 updates: [actions/setup-python](https://github.com/actions/setup-python) and [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `actions/setup-python` from 5.1.1 to 5.2.0 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) Updates `pypa/gh-action-pypi-publish` from 1.9.0 to 1.10.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index ec0601f0f4..9f115ba3e6 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -16,7 +16,7 @@ jobs: submodules: true fetch-depth: 0 - - uses: actions/setup-python@v5.1.1 + - uses: actions/setup-python@v5.2.0 name: Install Python with: python-version: '3.11' @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.9.0 + - uses: pypa/gh-action-pypi-publish@v1.10.0 with: user: __token__ password: ${{ secrets.pypi_password }} From 77f188c1f7b41e644aec3cb6eba72f3370228ba9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 11:41:19 +0100 Subject: [PATCH 0132/1043] Bump pypa/gh-action-pypi-publish from 1.9.0 to 1.10.0 in the actions group (#2147) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.9.0 to 1.10.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.9.0...v1.10.0) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index b54cbe48b3..1783afd806 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -55,7 +55,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.9.0 + - uses: pypa/gh-action-pypi-publish@v1.10.0 with: user: __token__ password: ${{ secrets.pypi_password }} From 60b4f57943419d05d831de227ce58ea2fa1997d1 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Mon, 2 Sep 2024 18:00:39 +0300 Subject: [PATCH 0133/1043] chore: update pre-commit hook (#2148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.5.7 → v0.6.3](https://github.com/astral-sh/ruff-pre-commit/compare/v0.5.7...v0.6.3) --- .pre-commit-config.yaml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dbb6634129..14a6e6d5ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.5.7' + rev: v0.6.3 hooks: - id: ruff args: ["--fix", "--show-fixes"] diff --git a/pyproject.toml b/pyproject.toml index 8bee491a82..1e5716b6ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -197,6 +197,7 @@ extend-exclude = [ "buck-out", "build", "dist", + "notebooks", # temporary, until we achieve compatibility with ruff ≥ 0.6 "venv", "docs", "src/zarr/v2/", From f1978dde52b11a7803a1a06937e69948acc8fe84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 12:23:21 +0100 Subject: [PATCH 0134/1043] Bump numpy from 2.1.0 to 2.1.1 in the requirements group (#2151) Bumps the requirements group with 1 update: [numpy](https://github.com/numpy/numpy). Updates `numpy` from 2.1.0 to 2.1.1 - [Release notes](https://github.com/numpy/numpy/releases) - [Changelog](https://github.com/numpy/numpy/blob/main/doc/RELEASE_WALKTHROUGH.rst) - [Commits](https://github.com/numpy/numpy/compare/v2.1.0...v2.1.1) --- updated-dependencies: - dependency-name: numpy dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_numpy.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_numpy.txt b/requirements_dev_numpy.txt index 6423997a29..190610d3d3 100644 --- a/requirements_dev_numpy.txt +++ b/requirements_dev_numpy.txt @@ -1,4 +1,4 @@ # Break this out into a separate file to allow testing against # different versions of numpy. This file should pin to the latest # numpy version. -numpy==2.1.0 +numpy==2.1.1 From 3b793c1569351e201844400f95a8a6689a63a61f Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Wed, 4 Sep 2024 16:13:17 -0700 Subject: [PATCH 0135/1043] [v3] h5py compat methods on Group (#2128) * feature(h5compat): add create_dataset, require_dataset, require_group, and require_gruops methods to group class * make mypy happy * doc fixes * write initial tests * more tests * add deprecation warnings * add deprecation warnings * switch up test --- src/zarr/core/common.py | 1 + src/zarr/core/group.py | 251 +++++++++++++++++++++++++++++++++++++++- tests/v3/test_group.py | 87 ++++++++++++++ 3 files changed, 337 insertions(+), 2 deletions(-) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index aaa30cfcb8..99ab58fae9 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -28,6 +28,7 @@ ZATTRS_JSON = ".zattrs" BytesLike = bytes | bytearray | memoryview +ShapeLike = tuple[int, ...] | int ChunkCoords = tuple[int, ...] ChunkCoordsLike = Iterable[int] ZarrFormat = Literal[2, 3] diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 566acf7ab8..8dbad2a496 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -7,6 +7,7 @@ from dataclasses import asdict, dataclass, field, replace from typing import TYPE_CHECKING, Literal, cast, overload +import numpy as np import numpy.typing as npt from typing_extensions import deprecated @@ -25,6 +26,7 @@ ZGROUP_JSON, ChunkCoords, ZarrFormat, + parse_shapelike, ) from zarr.core.config import config from zarr.core.sync import SyncMixin, sync @@ -250,7 +252,7 @@ async def getitem( if zarray is not None: # TODO: update this once the V2 array support is part of the primary array class zarr_json = {**zarray, "attributes": zattrs} - return AsyncArray.from_dict(store_path, zarray) + return AsyncArray.from_dict(store_path, zarr_json) else: zgroup = ( json.loads(zgroup_bytes.to_bytes()) @@ -324,6 +326,42 @@ async def create_group( zarr_format=self.metadata.zarr_format, ) + async def require_group(self, name: str, overwrite: bool = False) -> AsyncGroup: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : string + Group name. + overwrite : bool, optional + Overwrite any existing group with given `name` if present. + + Returns + ------- + g : AsyncGroup + """ + if overwrite: + # TODO: check that exists_ok=True errors if an array exists where the group is being created + grp = await self.create_group(name, exists_ok=True) + else: + try: + item: AsyncGroup | AsyncArray = await self.getitem(name) + if not isinstance(item, AsyncGroup): + raise TypeError( + f"Incompatible object ({item.__class__.__name__}) already exists" + ) + assert isinstance(item, AsyncGroup) # make mypy happy + grp = item + except KeyError: + grp = await self.create_group(name) + return grp + + async def require_groups(self, *names: str) -> tuple[AsyncGroup, ...]: + """Convenience method to require multiple groups in a single call.""" + if not names: + return () + return tuple(await asyncio.gather(*(self.require_group(name) for name in names))) + async def create_array( self, name: str, @@ -413,6 +451,117 @@ async def create_array( data=data, ) + @deprecated("Use AsyncGroup.create_array instead.") + async def create_dataset(self, name: str, **kwargs: Any) -> AsyncArray: + """Create an array. + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.AsyncGroup.require_dataset` method. + + Parameters + ---------- + name : string + Array name. + kwargs : dict + Additional arguments passed to :func:`zarr.AsyncGroup.create_array`. + + Returns + ------- + a : AsyncArray + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `AsyncGroup.create_array` instead. + """ + return await self.create_array(name, **kwargs) + + @deprecated("Use AsyncGroup.require_array instead.") + async def require_dataset( + self, + name: str, + *, + shape: ChunkCoords, + dtype: npt.DTypeLike = None, + exact: bool = False, + **kwargs: Any, + ) -> AsyncArray: + """Obtain an array, creating if it doesn't exist. + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.AsyncGroup.create_dataset` method. + + Other `kwargs` are as per :func:`zarr.AsyncGroup.create_dataset`. + + Parameters + ---------- + name : string + Array name. + shape : int or tuple of ints + Array shape. + dtype : string or dtype, optional + NumPy dtype. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + + Returns + ------- + a : AsyncArray + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `AsyncGroup.require_dataset` instead. + """ + return await self.require_array(name, shape=shape, dtype=dtype, exact=exact, **kwargs) + + async def require_array( + self, + name: str, + *, + shape: ChunkCoords, + dtype: npt.DTypeLike = None, + exact: bool = False, + **kwargs: Any, + ) -> AsyncArray: + """Obtain an array, creating if it doesn't exist. + + Other `kwargs` are as per :func:`zarr.AsyncGroup.create_dataset`. + + Parameters + ---------- + name : string + Array name. + shape : int or tuple of ints + Array shape. + dtype : string or dtype, optional + NumPy dtype. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + + Returns + ------- + a : AsyncArray + """ + try: + ds = await self.getitem(name) + if not isinstance(ds, AsyncArray): + raise TypeError(f"Incompatible object ({ds.__class__.__name__}) already exists") + + shape = parse_shapelike(shape) + if shape != ds.shape: + raise TypeError(f"Incompatible shape ({ds.shape} vs {shape})") + + dtype = np.dtype(dtype) + if exact: + if ds.dtype != dtype: + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + else: + if not np.can_cast(ds.dtype, dtype): + raise TypeError(f"Incompatible dtype ({ds.dtype} vs {dtype})") + except KeyError: + ds = await self.create_array(name, shape=shape, dtype=dtype, **kwargs) + + return ds + async def update_attributes(self, new_attributes: dict[str, Any]) -> AsyncGroup: # metadata.attributes is "frozen" so we simply clear and update the dict self.metadata.attributes.clear() @@ -612,8 +761,9 @@ def create( def open( cls, store: StoreLike, + zarr_format: Literal[2, 3, None] = 3, ) -> Group: - obj = sync(AsyncGroup.open(store)) + obj = sync(AsyncGroup.open(store, zarr_format=zarr_format)) return cls(obj) def __getitem__(self, path: str) -> Array | Group: @@ -717,6 +867,26 @@ def tree(self, expand: bool = False, level: int | None = None) -> Any: def create_group(self, name: str, **kwargs: Any) -> Group: return Group(self._sync(self._async_group.create_group(name, **kwargs))) + def require_group(self, name: str, **kwargs: Any) -> Group: + """Obtain a sub-group, creating one if it doesn't exist. + + Parameters + ---------- + name : string + Group name. + overwrite : bool, optional + Overwrite any existing group with given `name` if present. + + Returns + ------- + g : Group + """ + return Group(self._sync(self._async_group.require_group(name, **kwargs))) + + def require_groups(self, *names: str) -> tuple[Group, ...]: + """Convenience method to require multiple groups in a single call.""" + return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) + def create_array( self, name: str, @@ -811,6 +981,83 @@ def create_array( ) ) + @deprecated("Use Group.create_array instead.") + def create_dataset(self, name: str, **kwargs: Any) -> Array: + """Create an array. + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.Group.require_dataset` method. + + Parameters + ---------- + name : string + Array name. + kwargs : dict + Additional arguments passed to :func:`zarr.Group.create_array` + + Returns + ------- + a : Array + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `Group.create_array` instead. + """ + return Array(self._sync(self._async_group.create_dataset(name, **kwargs))) + + @deprecated("Use Group.require_array instead.") + def require_dataset(self, name: str, **kwargs: Any) -> Array: + """Obtain an array, creating if it doesn't exist. + + Arrays are known as "datasets" in HDF5 terminology. For compatibility + with h5py, Zarr groups also implement the :func:`zarr.Group.create_dataset` method. + + Other `kwargs` are as per :func:`zarr.Group.create_dataset`. + + Parameters + ---------- + name : string + Array name. + shape : int or tuple of ints + Array shape. + dtype : string or dtype, optional + NumPy dtype. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + + Returns + ------- + a : Array + + .. deprecated:: 3.0.0 + The h5py compatibility methods will be removed in 3.1.0. Use `Group.require_array` instead. + """ + return Array(self._sync(self._async_group.require_array(name, **kwargs))) + + def require_array(self, name: str, **kwargs: Any) -> Array: + """Obtain an array, creating if it doesn't exist. + + + Other `kwargs` are as per :func:`zarr.Group.create_array`. + + Parameters + ---------- + name : string + Array name. + shape : int or tuple of ints + Array shape. + dtype : string or dtype, optional + NumPy dtype. + exact : bool, optional + If True, require `dtype` to match exactly. If false, require + `dtype` can be cast from array dtype. + + Returns + ------- + a : Array + """ + return Array(self._sync(self._async_group.require_array(name, **kwargs))) + def empty(self, **kwargs: Any) -> Array: return Array(self._sync(self._async_group.empty(**kwargs))) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index fa0df3673e..80a06febec 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -730,3 +730,90 @@ async def test_group_members_async(store: LocalStore | MemoryStore): with pytest.raises(ValueError, match="max_depth"): [x async for x in group.members(max_depth=-1)] + + +async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + + # create foo group + _ = await root.create_group("foo", attributes={"foo": 100}) + + # test that we can get the group using require_group + foo_group = await root.require_group("foo") + assert foo_group.attrs == {"foo": 100} + + # test that we can get the group using require_group and overwrite=True + foo_group = await root.require_group("foo", overwrite=True) + + _ = await foo_group.create_array( + "bar", shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} + ) + + # test that overwriting a group w/ children fails + # TODO: figure out why ensure_no_existing_node is not catching the foo.bar array + # + # with pytest.raises(ContainsArrayError): + # await root.require_group("foo", overwrite=True) + + # test that requiring a group where an array is fails + with pytest.raises(TypeError): + await foo_group.require_group("bar") + + +async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + # create foo group + _ = await root.create_group("foo", attributes={"foo": 100}) + # create bar group + _ = await root.create_group("bar", attributes={"bar": 200}) + + foo_group, bar_group = await root.require_groups("foo", "bar") + assert foo_group.attrs == {"foo": 100} + assert bar_group.attrs == {"bar": 200} + + # get a mix of existing and new groups + foo_group, spam_group = await root.require_groups("foo", "spam") + assert foo_group.attrs == {"foo": 100} + assert spam_group.attrs == {} + + # no names + no_group = await root.require_groups() + assert no_group == () + + +async def test_create_dataset(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + foo = await root.create_dataset("foo", shape=(10,), dtype="uint8") + assert foo.shape == (10,) + + with pytest.raises(ContainsArrayError): + await root.create_dataset("foo", shape=(100,), dtype="int8") + + _ = await root.create_group("bar") + with pytest.raises(ContainsGroupError): + await root.create_dataset("bar", shape=(100,), dtype="int8") + + +async def test_require_array(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + foo1 = await root.require_array("foo", shape=(10,), dtype="i8", attributes={"foo": 101}) + assert foo1.attrs == {"foo": 101} + foo2 = await root.require_array("foo", shape=(10,), dtype="i8") + assert foo2.attrs == {"foo": 101} + + # exact = False + _ = await root.require_array("foo", shape=10, dtype="f8") + + # errors w/ exact True + with pytest.raises(TypeError, match="Incompatible dtype"): + await root.require_array("foo", shape=(10,), dtype="f8", exact=True) + + with pytest.raises(TypeError, match="Incompatible shape"): + await root.require_array("foo", shape=(100, 100), dtype="i8") + + with pytest.raises(TypeError, match="Incompatible dtype"): + await root.require_array("foo", shape=(10,), dtype="f4") + + _ = await root.create_group("bar") + with pytest.raises(TypeError, match="Incompatible object"): + await root.require_array("bar", shape=(10,), dtype="int8") From 88b2100ed3f0fed4c5b0a0bc38d29348d44e1ea0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Thu, 5 Sep 2024 05:40:20 -0700 Subject: [PATCH 0136/1043] chore(docs): update release notes ahead of 2.18.4 (#2152) --- docs/release.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 69616d6052..a62d6a653c 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,19 @@ Release notes See `GH1777 `_ for more details on the upcoming 3.0 release. +.. _release_2.18.4: + +2.18.4 (unreleased) +------------------- + +Enhancements +~~~~~~~~~~~~ + +Maintenance +~~~~~~~~~~~ + +Deprecations +~~~~~~~~~~~~ .. _release_2.18.3: From de075de09fd31eb5417bc3faf71f8156c61e3bda Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:52:43 +0300 Subject: [PATCH 0137/1043] Enforce Repo-Review rules (#2075) Co-authored-by: David Stansby --- .pre-commit-config.yaml | 9 +++++++++ pyproject.toml | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 14a6e6d5ef..3c0c4f9f92 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,3 +39,12 @@ repos: - pytest # Zarr v2 - types-redis + - repo: https://github.com/scientific-python/cookie + rev: 2024.04.23 + hooks: + - id: sp-repo-review + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: rst-directive-colons + - id: rst-inline-touching-normal diff --git a/pyproject.toml b/pyproject.toml index 1e5716b6ca..accb4acd12 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -258,3 +258,9 @@ filterwarnings = [ markers = [ "gpu: mark a test as requiring CuPy and GPU" ] + +[tool.repo-review] +ignore = [ + "PC111", # fix Python code in documentation - enable later + "PC180", # for JavaScript - not interested +] From c08b00865fb17ea41881b23efc739fca6ecfdf1e Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 6 Sep 2024 16:10:54 +0200 Subject: [PATCH 0138/1043] Enforce ruff/flake8-type-checking rules (TCH) (#2110) * Apply ruff/flake8-type-checking rule TCH001 TCH001 Move application import into a type-checking block * Apply ruff/flake8-type-checking rule TCH002 TCH002 Move third-party import into a type-checking block * Apply ruff/flake8-type-checking rule TCH003 TCH003 Move standard library import into a type-checking block * Apply ruff/flake8-type-checking rule TCH005 TCH005 Found empty type-checking block * Enforce ruff/flake8-type-checking rules (TCH) --- pyproject.toml | 1 + src/zarr/abc/codec.py | 10 +++++----- src/zarr/abc/metadata.py | 4 ++-- src/zarr/api/asynchronous.py | 13 ++++++++----- src/zarr/api/synchronous.py | 10 ++++++---- src/zarr/codecs/_v2.py | 5 ++++- src/zarr/codecs/blosc.py | 5 +++-- src/zarr/codecs/bytes.py | 3 ++- src/zarr/codecs/crc32c_.py | 5 +++-- src/zarr/codecs/gzip.py | 5 +++-- src/zarr/codecs/pipeline.py | 9 ++++----- src/zarr/codecs/transpose.py | 5 +++-- src/zarr/codecs/zstd.py | 5 +++-- src/zarr/core/array.py | 9 ++++++--- src/zarr/core/array_spec.py | 4 ++-- src/zarr/core/buffer/core.py | 6 +++--- src/zarr/core/buffer/cpu.py | 5 +++-- src/zarr/core/buffer/gpu.py | 3 ++- src/zarr/core/chunk_grids.py | 3 ++- src/zarr/core/chunk_key_encodings.py | 5 +---- src/zarr/core/group.py | 7 +++---- src/zarr/core/indexing.py | 2 +- src/zarr/store/local.py | 3 ++- src/zarr/store/memory.py | 3 ++- src/zarr/store/remote.py | 6 +++--- src/zarr/testing/buffer.py | 3 ++- src/zarr/testing/utils.py | 6 ++++-- tests/v3/conftest.py | 13 +++++++------ tests/v3/test_codecs/test_codecs.py | 9 ++++++--- tests/v3/test_codecs/test_transpose.py | 6 +++++- tests/v3/test_common.py | 2 +- tests/v3/test_group.py | 6 ++++-- tests/v3/test_indexing.py | 11 +++++++---- tests/v3/test_metadata/test_v2.py | 4 ++-- tests/v3/test_metadata/test_v3.py | 5 +++-- 35 files changed, 118 insertions(+), 83 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index accb4acd12..6c9ed4521e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -212,6 +212,7 @@ extend-select = [ "UP", # pyupgrade "RSE", "RUF", + "TCH", # flake8-type-checking "TRY", # tryceratops ] ignore = [ diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index fa76399f90..cc32b9bcfc 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -1,22 +1,22 @@ from __future__ import annotations from abc import abstractmethod -from collections.abc import Awaitable, Callable, Iterable from typing import TYPE_CHECKING, Any, Generic, TypeVar -import numpy as np - from zarr.abc.metadata import Metadata -from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.buffer import Buffer, NDBuffer -from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import ChunkCoords, concurrent_map from zarr.core.config import config if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Iterable + + import numpy as np from typing_extensions import Self + from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec + from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import JSON from zarr.core.indexing import SelectorTuple diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py index 0ddf762766..d9b11af883 100644 --- a/src/zarr/abc/metadata.py +++ b/src/zarr/abc/metadata.py @@ -6,9 +6,9 @@ if TYPE_CHECKING: from typing_extensions import Self -from dataclasses import dataclass, fields + from zarr.core.common import JSON -from zarr.core.common import JSON +from dataclasses import dataclass, fields __all__ = ["Metadata"] diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index ad89584b44..7f59517f39 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -2,16 +2,12 @@ import asyncio import warnings -from collections.abc import Iterable -from typing import Any, Literal, Union, cast +from typing import TYPE_CHECKING, Any, Literal, Union, cast import numpy as np import numpy.typing as npt -from zarr.abc.codec import Codec from zarr.core.array import Array, AsyncArray -from zarr.core.buffer import NDArrayLike -from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat from zarr.core.group import AsyncGroup from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata @@ -20,6 +16,13 @@ make_store_path, ) +if TYPE_CHECKING: + from collections.abc import Iterable + + from zarr.abc.codec import Codec + from zarr.core.buffer import NDArrayLike + from zarr.core.chunk_key_encodings import ChunkKeyEncoding + __all__ = [ "consolidate_metadata", "copy", diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index d3d674c3fd..93a33b8d3f 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -1,14 +1,16 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any import zarr.api.asynchronous as async_api from zarr.core.array import Array, AsyncArray -from zarr.core.buffer import NDArrayLike -from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, ZarrFormat from zarr.core.group import Group from zarr.core.sync import sync -from zarr.store import StoreLike + +if TYPE_CHECKING: + from zarr.core.buffer import NDArrayLike + from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, ZarrFormat + from zarr.store import StoreLike __all__ = [ "consolidate_metadata", diff --git a/src/zarr/codecs/_v2.py b/src/zarr/codecs/_v2.py index 3dc5c8e4a7..eb8ec435f5 100644 --- a/src/zarr/codecs/_v2.py +++ b/src/zarr/codecs/_v2.py @@ -1,16 +1,19 @@ from __future__ import annotations from dataclasses import dataclass +from typing import TYPE_CHECKING import numcodecs from numcodecs.compat import ensure_bytes, ensure_ndarray from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec -from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer, default_buffer_prototype from zarr.core.common import JSON, to_thread from zarr.registry import get_ndbuffer_class +if TYPE_CHECKING: + from zarr.core.array_spec import ArraySpec + @dataclass(frozen=True) class V2Compressor(ArrayBytesCodec): diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index ffe5a0b6f3..cd265e3dc0 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -9,8 +9,6 @@ from numcodecs.blosc import Blosc from zarr.abc.codec import BytesBytesCodec -from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import Buffer from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_enum, parse_named_configuration, to_thread from zarr.registry import register_codec @@ -18,6 +16,9 @@ if TYPE_CHECKING: from typing_extensions import Self + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + class BloscShuffle(Enum): noshuffle = "noshuffle" diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index c2ef0ee4f6..4561b22346 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -8,7 +8,6 @@ import numpy as np from zarr.abc.codec import ArrayBytesCodec -from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDArrayLike, NDBuffer from zarr.core.common import JSON, parse_enum, parse_named_configuration from zarr.registry import register_codec @@ -16,6 +15,8 @@ if TYPE_CHECKING: from typing_extensions import Self + from zarr.core.array_spec import ArraySpec + class Endian(Enum): big = "big" diff --git a/src/zarr/codecs/crc32c_.py b/src/zarr/codecs/crc32c_.py index a519c1ba79..f814ba15e5 100644 --- a/src/zarr/codecs/crc32c_.py +++ b/src/zarr/codecs/crc32c_.py @@ -8,14 +8,15 @@ from crc32c import crc32c from zarr.abc.codec import BytesBytesCodec -from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import Buffer from zarr.core.common import JSON, parse_named_configuration from zarr.registry import register_codec if TYPE_CHECKING: from typing_extensions import Self + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + @dataclass(frozen=True) class Crc32cCodec(BytesBytesCodec): diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py index d8d793a965..0dd31009c4 100644 --- a/src/zarr/codecs/gzip.py +++ b/src/zarr/codecs/gzip.py @@ -6,8 +6,6 @@ from numcodecs.gzip import GZip from zarr.abc.codec import BytesBytesCodec -from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import Buffer from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration, to_thread from zarr.registry import register_codec @@ -15,6 +13,9 @@ if TYPE_CHECKING: from typing_extensions import Self + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + def parse_gzip_level(data: JSON) -> int: if not isinstance(data, (int)): diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/codecs/pipeline.py index 9c615d3d2c..3a400d6eaf 100644 --- a/src/zarr/codecs/pipeline.py +++ b/src/zarr/codecs/pipeline.py @@ -6,8 +6,6 @@ from typing import TYPE_CHECKING, Any, TypeVar from warnings import warn -import numpy as np - from zarr.abc.codec import ( ArrayArrayCodec, ArrayBytesCodec, @@ -17,18 +15,19 @@ Codec, CodecPipeline, ) -from zarr.abc.store import ByteGetter, ByteSetter -from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer -from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import JSON, ChunkCoords, concurrent_map, parse_named_configuration from zarr.core.config import config from zarr.core.indexing import SelectorTuple, is_scalar, is_total_slice from zarr.registry import get_codec_class, register_pipeline if TYPE_CHECKING: + import numpy as np from typing_extensions import Self + from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer, BufferPrototype, NDBuffer + from zarr.core.chunk_grids import ChunkGrid T = TypeVar("T") U = TypeVar("U") diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index ada45b1b53..9bb795a3a1 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -8,8 +8,6 @@ from zarr.abc.codec import ArrayArrayCodec from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import NDBuffer -from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import JSON, ChunkCoordsLike, parse_named_configuration from zarr.registry import register_codec @@ -18,6 +16,9 @@ from typing_extensions import Self + from zarr.core.buffer import NDBuffer + from zarr.core.chunk_grids import ChunkGrid + def parse_transpose_order(data: JSON | Iterable[int]) -> tuple[int, ...]: if not isinstance(data, Iterable): diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index fceb49cf2b..572d594d53 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -8,8 +8,6 @@ from numcodecs.zstd import Zstd from zarr.abc.codec import BytesBytesCodec -from zarr.core.array_spec import ArraySpec -from zarr.core.buffer import Buffer from zarr.core.buffer.cpu import as_numpy_array_wrapper from zarr.core.common import JSON, parse_named_configuration, to_thread from zarr.registry import register_codec @@ -17,6 +15,9 @@ if TYPE_CHECKING: from typing_extensions import Self + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer + def parse_zstd_level(data: JSON) -> int: if isinstance(data, int): diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index e7b5709b77..ddf3a09c33 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -2,14 +2,12 @@ import json from asyncio import gather -from collections.abc import Iterable from dataclasses import dataclass, field, replace -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np import numpy.typing as npt -from zarr.abc.codec import Codec, CodecPipeline from zarr.abc.store import set_or_delete from zarr.codecs import BytesCodec from zarr.codecs._v2 import V2Compressor, V2Filters @@ -63,6 +61,11 @@ ensure_no_existing_node, ) +if TYPE_CHECKING: + from collections.abc import Iterable + + from zarr.abc.codec import Codec, CodecPipeline + def parse_array_metadata(data: Any) -> ArrayV2Metadata | ArrayV3Metadata: if isinstance(data, ArrayV2Metadata | ArrayV3Metadata): diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py index d2f46057e4..e64a962bc3 100644 --- a/src/zarr/core/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -3,11 +3,11 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal -import numpy as np - from zarr.core.common import parse_dtype, parse_fill_value, parse_order, parse_shapelike if TYPE_CHECKING: + import numpy as np + from zarr.core.buffer import BufferPrototype from zarr.core.common import ChunkCoords diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 6e5d9465cf..fca2da2b77 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -2,7 +2,6 @@ import sys from abc import ABC, abstractmethod -from collections.abc import Iterable, Sequence from typing import ( TYPE_CHECKING, Any, @@ -17,17 +16,18 @@ import numpy as np import numpy.typing as npt -from zarr.core.common import ChunkCoords from zarr.registry import ( get_buffer_class, get_ndbuffer_class, ) if TYPE_CHECKING: + from collections.abc import Iterable, Sequence + from typing_extensions import Self from zarr.codecs.bytes import Endian - from zarr.core.common import BytesLike + from zarr.core.common import BytesLike, ChunkCoords @runtime_checkable diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 0953805223..66d5ecfe7c 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import Callable, Iterable from typing import ( TYPE_CHECKING, Any, @@ -11,15 +10,17 @@ import numpy.typing as npt from zarr.core.buffer import core -from zarr.core.buffer.core import ArrayLike, NDArrayLike from zarr.registry import ( register_buffer, register_ndbuffer, ) if TYPE_CHECKING: + from collections.abc import Callable, Iterable + from typing_extensions import Self + from zarr.core.buffer.core import ArrayLike, NDArrayLike from zarr.core.common import BytesLike diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index 9d38df2abf..476a4a62c1 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -1,7 +1,6 @@ from __future__ import annotations import warnings -from collections.abc import Iterable from typing import ( TYPE_CHECKING, Any, @@ -16,6 +15,8 @@ from zarr.core.buffer.core import ArrayLike, BufferPrototype, NDArrayLike if TYPE_CHECKING: + from collections.abc import Iterable + from typing_extensions import Self from zarr.core.common import BytesLike diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index f912872251..abdd2688f9 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -4,7 +4,6 @@ import math import operator from abc import abstractmethod -from collections.abc import Iterator from dataclasses import dataclass from functools import reduce from typing import TYPE_CHECKING @@ -22,6 +21,8 @@ from zarr.core.indexing import ceildiv if TYPE_CHECKING: + from collections.abc import Iterator + from typing_extensions import Self diff --git a/src/zarr/core/chunk_key_encodings.py b/src/zarr/core/chunk_key_encodings.py index 45174bf4af..ed12ee3065 100644 --- a/src/zarr/core/chunk_key_encodings.py +++ b/src/zarr/core/chunk_key_encodings.py @@ -2,7 +2,7 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import TYPE_CHECKING, Literal, cast +from typing import Literal, cast from zarr.abc.metadata import Metadata from zarr.core.common import ( @@ -11,9 +11,6 @@ parse_named_configuration, ) -if TYPE_CHECKING: - pass - SeparatorLiteral = Literal[".", "/"] diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 8dbad2a496..89859ab6f8 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -3,7 +3,6 @@ import asyncio import json import logging -from collections.abc import Iterator from dataclasses import asdict, dataclass, field, replace from typing import TYPE_CHECKING, Literal, cast, overload @@ -11,13 +10,11 @@ import numpy.typing as npt from typing_extensions import deprecated -from zarr.abc.codec import Codec from zarr.abc.metadata import Metadata from zarr.abc.store import set_or_delete from zarr.core.array import Array, AsyncArray from zarr.core.attributes import Attributes from zarr.core.buffer import default_buffer_prototype -from zarr.core.chunk_key_encodings import ChunkKeyEncoding from zarr.core.common import ( JSON, ZARR_JSON, @@ -34,10 +31,12 @@ from zarr.store.common import ensure_no_existing_node if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Iterable + from collections.abc import AsyncGenerator, Iterable, Iterator from typing import Any + from zarr.abc.codec import Codec from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.chunk_key_encodings import ChunkKeyEncoding logger = logging.getLogger("zarr.group") diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 153483330c..e6e04e58f6 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -23,11 +23,11 @@ import numpy as np import numpy.typing as npt -from zarr.core.buffer import NDArrayLike from zarr.core.common import product if TYPE_CHECKING: from zarr.core.array import Array + from zarr.core.buffer import NDArrayLike from zarr.core.chunk_grids import ChunkGrid from zarr.core.common import ChunkCoords diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index 6510305844..b69b86dc4c 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -3,7 +3,6 @@ import io import os import shutil -from collections.abc import AsyncGenerator from pathlib import Path from typing import TYPE_CHECKING @@ -12,6 +11,8 @@ from zarr.core.common import concurrent_map, to_thread if TYPE_CHECKING: + from collections.abc import AsyncGenerator + from zarr.core.buffer import BufferPrototype from zarr.core.common import AccessModeLiteral diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index c817b0963a..e0e72b1827 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import AsyncGenerator, MutableMapping from typing import TYPE_CHECKING from zarr.abc.store import Store @@ -9,6 +8,8 @@ from zarr.store._utils import _normalize_interval_index if TYPE_CHECKING: + from collections.abc import AsyncGenerator, MutableMapping + from zarr.core.buffer import BufferPrototype from zarr.core.common import AccessModeLiteral diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index 83393e4dac..f883832bc7 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -1,19 +1,19 @@ from __future__ import annotations -from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any import fsspec from zarr.abc.store import Store -from zarr.core.buffer import Buffer from zarr.store.common import _dereference_path if TYPE_CHECKING: + from collections.abc import AsyncGenerator + from fsspec.asyn import AsyncFileSystem from upath import UPath - from zarr.core.buffer import BufferPrototype + from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.common import AccessModeLiteral, BytesLike diff --git a/src/zarr/testing/buffer.py b/src/zarr/testing/buffer.py index a6120ef2f9..119ffc0a66 100644 --- a/src/zarr/testing/buffer.py +++ b/src/zarr/testing/buffer.py @@ -1,7 +1,6 @@ # mypy: ignore-errors from __future__ import annotations -from collections.abc import Iterable from typing import TYPE_CHECKING, Any, Literal import numpy as np @@ -11,6 +10,8 @@ from zarr.store import MemoryStore if TYPE_CHECKING: + from collections.abc import Iterable + from typing_extensions import Self diff --git a/src/zarr/testing/utils.py b/src/zarr/testing/utils.py index a21a0be708..9d6dfa7e18 100644 --- a/src/zarr/testing/utils.py +++ b/src/zarr/testing/utils.py @@ -1,11 +1,13 @@ from __future__ import annotations -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import pytest from zarr.core.buffer import Buffer -from zarr.core.common import BytesLike + +if TYPE_CHECKING: + from zarr.core.common import BytesLike __all__ = ["assert_bytes_equal"] diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index c67d7bb4fd..fcb5e3c867 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -1,17 +1,18 @@ from __future__ import annotations -from collections.abc import Iterator -from types import ModuleType from typing import TYPE_CHECKING -from _pytest.compat import LEGACY_PATH - from zarr import AsyncGroup, config -from zarr.abc.store import Store -from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat if TYPE_CHECKING: + from collections.abc import Iterator + from types import ModuleType from typing import Any, Literal + + from _pytest.compat import LEGACY_PATH + + from zarr.abc.store import Store + from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat import pathlib from dataclasses import dataclass, field diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/v3/test_codecs/test_codecs.py index 7e717e5b50..f388fb1c1e 100644 --- a/tests/v3/test_codecs/test_codecs.py +++ b/tests/v3/test_codecs/test_codecs.py @@ -2,14 +2,13 @@ import json from dataclasses import dataclass +from typing import TYPE_CHECKING import numpy as np import pytest import zarr.v2 from zarr import Array, AsyncArray, config -from zarr.abc.codec import Codec -from zarr.abc.store import Store from zarr.codecs import ( BytesCodec, GzipCodec, @@ -17,11 +16,15 @@ TransposeCodec, ) from zarr.core.buffer import default_buffer_prototype -from zarr.core.common import MemoryOrder from zarr.core.indexing import Selection, morton_order_iter from zarr.store import StorePath from zarr.testing.utils import assert_bytes_equal +if TYPE_CHECKING: + from zarr.abc.codec import Codec + from zarr.abc.store import Store + from zarr.core.common import MemoryOrder + @dataclass(frozen=True) class _AsyncArrayProxy: diff --git a/tests/v3/test_codecs/test_transpose.py b/tests/v3/test_codecs/test_transpose.py index b9aa6cc4b6..a24c650c5b 100644 --- a/tests/v3/test_codecs/test_transpose.py +++ b/tests/v3/test_codecs/test_transpose.py @@ -1,9 +1,10 @@ +from typing import TYPE_CHECKING + import numpy as np import pytest import zarr.v2 from zarr import Array, AsyncArray, config -from zarr.abc.codec import Codec from zarr.abc.store import Store from zarr.codecs import BytesCodec, ShardingCodec, TransposeCodec from zarr.core.buffer import default_buffer_prototype @@ -12,6 +13,9 @@ from .test_codecs import _AsyncArrayProxy +if TYPE_CHECKING: + from zarr.abc.codec import Codec + @pytest.mark.parametrize("input_order", ["F", "C"]) @pytest.mark.parametrize("runtime_write_order", ["F", "C"]) diff --git a/tests/v3/test_common.py b/tests/v3/test_common.py index f919b4d5f7..85c6d93d36 100644 --- a/tests/v3/test_common.py +++ b/tests/v3/test_common.py @@ -1,9 +1,9 @@ from __future__ import annotations -from collections.abc import Iterable from typing import TYPE_CHECKING if TYPE_CHECKING: + from collections.abc import Iterable from typing import Any, Literal import numpy as np diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 80a06febec..e588b6a11b 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,10 +1,9 @@ from __future__ import annotations -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np import pytest -from _pytest.compat import LEGACY_PATH from zarr import Array, AsyncArray, AsyncGroup, Group from zarr.core.buffer import default_buffer_prototype @@ -17,6 +16,9 @@ from .conftest import parse_store +if TYPE_CHECKING: + from _pytest.compat import LEGACY_PATH + @pytest.fixture(params=["local", "memory"]) async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> LocalStore | MemoryStore: diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 5805de3035..3230bbc675 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -1,8 +1,7 @@ from __future__ import annotations from collections import Counter -from collections.abc import Iterator -from typing import Any +from typing import TYPE_CHECKING, Any from uuid import uuid4 import numpy as np @@ -11,9 +10,7 @@ from numpy.testing import assert_array_equal import zarr -from zarr.abc.store import Store from zarr.core.buffer import BufferPrototype, default_buffer_prototype -from zarr.core.common import ChunkCoords from zarr.core.indexing import ( make_slice_selection, normalize_integer_selection, @@ -25,6 +22,12 @@ from zarr.store.common import StorePath from zarr.store.memory import MemoryStore +if TYPE_CHECKING: + from collections.abc import Iterator + + from zarr.abc.store import Store + from zarr.core.common import ChunkCoords + @pytest.fixture async def store() -> Iterator[Store]: diff --git a/tests/v3/test_metadata/test_v2.py b/tests/v3/test_metadata/test_v2.py index 54f26d91ca..0053de08ca 100644 --- a/tests/v3/test_metadata/test_v2.py +++ b/tests/v3/test_metadata/test_v2.py @@ -2,11 +2,11 @@ from typing import TYPE_CHECKING, Literal -from zarr.abc.codec import Codec - if TYPE_CHECKING: from typing import Any + from zarr.abc.codec import Codec + import pytest from zarr.codecs import GzipCodec diff --git a/tests/v3/test_metadata/test_v3.py b/tests/v3/test_metadata/test_v3.py index 1a0c5b94d7..bc43154a54 100644 --- a/tests/v3/test_metadata/test_v3.py +++ b/tests/v3/test_metadata/test_v3.py @@ -4,15 +4,16 @@ import re from typing import TYPE_CHECKING, Literal -from zarr.abc.codec import Codec from zarr.codecs.bytes import BytesCodec from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding if TYPE_CHECKING: + from collections.abc import Sequence from typing import Any -from collections.abc import Sequence + from zarr.abc.codec import Codec + import numpy as np import pytest From 5e113f5b43250e8dcb9f2dbb381b0dbccb39cc2a Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 6 Sep 2024 15:44:25 +0100 Subject: [PATCH 0139/1043] Add UPath dependency/type checking (#1955) Add universal-pathlib to deps Make universal-pathlib optional --- .pre-commit-config.yaml | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c0c4f9f92..e7901aa67b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,6 +35,7 @@ repos: - numcodecs - numpy - typing_extensions + - universal-pathlib # Tests - pytest # Zarr v2 diff --git a/pyproject.toml b/pyproject.toml index 6c9ed4521e..c6ab4a4638 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,6 +95,7 @@ extra = [ ] optional = [ 'lmdb', + 'universal-pathlib', ] [project.urls] From e8800b0d55596fc200d67ef2cb8e6f544dcbb519 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 6 Sep 2024 10:39:13 -0500 Subject: [PATCH 0140/1043] Check tests folder with mypy (#2150) * Check tests folder with mypy * use shapelike * fixup --- .pre-commit-config.yaml | 2 +- src/zarr/core/array.py | 14 ++++- src/zarr/core/chunk_grids.py | 5 +- src/zarr/core/group.py | 9 +-- tests/v3/package_with_entrypoint/__init__.py | 4 +- tests/v3/test_api.py | 60 +++++++++++--------- tests/v3/test_buffer.py | 26 +++++---- 7 files changed, 73 insertions(+), 47 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7901aa67b..11ec156d20 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: rev: v1.11.2 hooks: - id: mypy - files: src + files: src|tests/v3/test_(api|array|buffer).py additional_dependencies: # Package dependencies - asciitree diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index ddf3a09c33..fbe0b19f6b 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -25,8 +25,10 @@ ZARRAY_JSON, ZATTRS_JSON, ChunkCoords, + ShapeLike, ZarrFormat, concurrent_map, + parse_shapelike, product, ) from zarr.core.config import config, parse_indexing_order @@ -116,7 +118,7 @@ async def create( store: StoreLike, *, # v2 and v3 - shape: ChunkCoords, + shape: ShapeLike, dtype: npt.DTypeLike, zarr_format: ZarrFormat = 3, fill_value: Any | None = None, @@ -132,7 +134,7 @@ async def create( codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only - chunks: ChunkCoords | None = None, + chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, filters: list[dict[str, JSON]] | None = None, @@ -143,9 +145,14 @@ async def create( ) -> AsyncArray: store_path = await make_store_path(store) + shape = parse_shapelike(shape) + if chunk_shape is None: if chunks is None: chunk_shape = chunks = _guess_chunks(shape=shape, typesize=np.dtype(dtype).itemsize) + else: + chunks = parse_shapelike(chunks) + chunk_shape = chunks elif chunks is not None: raise ValueError("Only one of chunk_shape or chunks must be provided.") @@ -217,7 +224,7 @@ async def _create_v3( cls, store_path: StorePath, *, - shape: ChunkCoords, + shape: ShapeLike, dtype: npt.DTypeLike, chunk_shape: ChunkCoords, fill_value: Any | None = None, @@ -235,6 +242,7 @@ async def _create_v3( if not exists_ok: await ensure_no_existing_node(store_path, zarr_format=3) + shape = parse_shapelike(shape) codecs = list(codecs) if codecs is not None else [BytesCodec()] if fill_value is None: diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index abdd2688f9..61723215c6 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -15,6 +15,7 @@ JSON, ChunkCoords, ChunkCoordsLike, + ShapeLike, parse_named_configuration, parse_shapelike, ) @@ -27,7 +28,7 @@ def _guess_chunks( - shape: ChunkCoords, + shape: ShapeLike, typesize: int, *, increment_bytes: int = 256 * 1024, @@ -57,6 +58,8 @@ def _guess_chunks( ChunkCoords """ + if isinstance(shape, int): + shape = (shape,) ndims = len(shape) # require chunks to have non-zero length for all dimensions diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 89859ab6f8..56c9c88ea5 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -22,6 +22,7 @@ ZATTRS_JSON, ZGROUP_JSON, ChunkCoords, + ShapeLike, ZarrFormat, parse_shapelike, ) @@ -365,7 +366,7 @@ async def create_array( self, name: str, *, - shape: ChunkCoords, + shape: ShapeLike, dtype: npt.DTypeLike = "float64", fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -380,7 +381,7 @@ async def create_array( codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only - chunks: ChunkCoords | None = None, + chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, filters: list[dict[str, JSON]] | None = None, @@ -890,7 +891,7 @@ def create_array( self, name: str, *, - shape: ChunkCoords, + shape: ShapeLike, dtype: npt.DTypeLike = "float64", fill_value: Any | None = None, attributes: dict[str, JSON] | None = None, @@ -905,7 +906,7 @@ def create_array( codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, # v2 only - chunks: ChunkCoords | None = None, + chunks: ShapeLike | None = None, dimension_separator: Literal[".", "/"] | None = None, order: Literal["C", "F"] | None = None, filters: list[dict[str, JSON]] | None = None, diff --git a/tests/v3/package_with_entrypoint/__init__.py b/tests/v3/package_with_entrypoint/__init__.py index 0e84cde0d5..352b9d570e 100644 --- a/tests/v3/package_with_entrypoint/__init__.py +++ b/tests/v3/package_with_entrypoint/__init__.py @@ -2,7 +2,7 @@ from numpy import ndarray -from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecPipeline +from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecOutput, CodecPipeline from zarr.codecs import BytesCodec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import Buffer, NDBuffer @@ -15,7 +15,7 @@ class TestEntrypointCodec(ArrayBytesCodec): async def encode( self, chunks_and_specs: Iterable[tuple[CodecInput | None, ArraySpec]], - ) -> BytesLike | None: + ) -> Iterable[CodecOutput | None]: pass async def decode( diff --git a/tests/v3/test_api.py b/tests/v3/test_api.py index 002678be88..239dd1c3e2 100644 --- a/tests/v3/test_api.py +++ b/tests/v3/test_api.py @@ -1,12 +1,14 @@ +import pathlib + import numpy as np import pytest from numpy.testing import assert_array_equal -from pytest_asyncio import fixture import zarr from zarr import Array, Group from zarr.abc.store import Store from zarr.api.synchronous import create, load, open, open_group, save, save_array, save_group +from zarr.store.memory import MemoryStore def test_create_array(memory_store: Store) -> None: @@ -30,7 +32,7 @@ def test_create_array(memory_store: Store) -> None: assert z.chunks == (40,) -async def test_open_array(memory_store: Store) -> None: +async def test_open_array(memory_store: MemoryStore) -> None: store = memory_store # open array, create if doesn't exist @@ -57,7 +59,7 @@ async def test_open_array(memory_store: Store) -> None: open(store="doesnotexist", mode="r") -async def test_open_group(memory_store: Store) -> None: +async def test_open_group(memory_store: MemoryStore) -> None: store = memory_store # open group, create if doesn't exist @@ -85,59 +87,65 @@ def test_save_errors() -> None: save_group("data/group.zarr") with pytest.raises(TypeError): # no array provided - save_array("data/group.zarr") + save_array("data/group.zarr") # type: ignore[call-arg] with pytest.raises(ValueError): # no arrays provided save("data/group.zarr") -@fixture -def tmppath(tmpdir): - return str(tmpdir / "example.zarr") - - -def test_open_with_mode_r(tmppath) -> None: +def test_open_with_mode_r(tmp_path: pathlib.Path) -> None: # 'r' means read only (must exist) with pytest.raises(FileNotFoundError): - zarr.open(store=tmppath, mode="r") - zarr.ones(store=tmppath, shape=(3, 3)) - z2 = zarr.open(store=tmppath, mode="r") + zarr.open(store=tmp_path, mode="r") + zarr.ones(store=tmp_path, shape=(3, 3)) + z2 = zarr.open(store=tmp_path, mode="r") + assert isinstance(z2, Array) assert (z2[:] == 1).all() with pytest.raises(ValueError): z2[:] = 3 -def test_open_with_mode_r_plus(tmppath) -> None: +def test_open_with_mode_r_plus(tmp_path: pathlib.Path) -> None: # 'r+' means read/write (must exist) with pytest.raises(FileNotFoundError): - zarr.open(store=tmppath, mode="r+") - zarr.ones(store=tmppath, shape=(3, 3)) - z2 = zarr.open(store=tmppath, mode="r+") + zarr.open(store=tmp_path, mode="r+") + zarr.ones(store=tmp_path, shape=(3, 3)) + z2 = zarr.open(store=tmp_path, mode="r+") + assert isinstance(z2, Array) assert (z2[:] == 1).all() z2[:] = 3 -def test_open_with_mode_a(tmppath) -> None: +def test_open_with_mode_a(tmp_path: pathlib.Path) -> None: # 'a' means read/write (create if doesn't exist) - zarr.open(store=tmppath, mode="a", shape=(3, 3))[...] = 1 - z2 = zarr.open(store=tmppath, mode="a") + arr = zarr.open(store=tmp_path, mode="a", shape=(3, 3)) + assert isinstance(arr, Array) + arr[...] = 1 + z2 = zarr.open(store=tmp_path, mode="a") + assert isinstance(z2, Array) assert (z2[:] == 1).all() z2[:] = 3 -def test_open_with_mode_w(tmppath) -> None: +def test_open_with_mode_w(tmp_path: pathlib.Path) -> None: # 'w' means create (overwrite if exists); - zarr.open(store=tmppath, mode="w", shape=(3, 3))[...] = 3 - z2 = zarr.open(store=tmppath, mode="w", shape=(3, 3)) + arr = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) + assert isinstance(arr, Array) + + arr[...] = 3 + z2 = zarr.open(store=tmp_path, mode="w", shape=(3, 3)) + assert isinstance(z2, Array) assert not (z2[:] == 3).all() z2[:] = 3 -def test_open_with_mode_w_minus(tmppath) -> None: +def test_open_with_mode_w_minus(tmp_path: pathlib.Path) -> None: # 'w-' means create (fail if exists) - zarr.open(store=tmppath, mode="w-", shape=(3, 3))[...] = 1 + arr = zarr.open(store=tmp_path, mode="w-", shape=(3, 3)) + assert isinstance(arr, Array) + arr[...] = 1 with pytest.raises(FileExistsError): - zarr.open(store=tmppath, mode="w-") + zarr.open(store=tmp_path, mode="w-") # def test_lazy_loader(): diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 419f01c720..06c0714333 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -1,5 +1,7 @@ from __future__ import annotations +import types + import numpy as np import pytest @@ -27,14 +29,14 @@ cp = None -def test_nd_array_like(xp): +def test_nd_array_like(xp: types.ModuleType) -> None: ary = xp.arange(10) assert isinstance(ary, ArrayLike) assert isinstance(ary, NDArrayLike) @pytest.mark.asyncio -async def test_async_array_prototype(): +async def test_async_array_prototype() -> None: """Test the use of a custom buffer prototype""" expect = np.zeros((9, 9), dtype="uint16", order="F") @@ -55,13 +57,15 @@ async def test_async_array_prototype(): prototype=my_prototype, ) got = await a.getitem(selection=(slice(0, 9), slice(0, 9)), prototype=my_prototype) - assert isinstance(got, TestNDArrayLike) - assert np.array_equal(expect, got) + # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol + # The test passes, so it clearly does. + assert isinstance(got, TestNDArrayLike) # type: ignore[unreachable] + assert np.array_equal(expect, got) # type: ignore[unreachable] @gpu_test @pytest.mark.asyncio -async def test_async_array_gpu_prototype(): +async def test_async_array_gpu_prototype() -> None: """Test the use of the GPU buffer prototype""" expect = cp.zeros((9, 9), dtype="uint16", order="F") @@ -85,7 +89,7 @@ async def test_async_array_gpu_prototype(): @pytest.mark.asyncio -async def test_codecs_use_of_prototype(): +async def test_codecs_use_of_prototype() -> None: expect = np.zeros((10, 10), dtype="uint16", order="F") a = await AsyncArray.create( StorePath(StoreExpectingTestBuffer(mode="w")) / "test_codecs_use_of_prototype", @@ -112,13 +116,15 @@ async def test_codecs_use_of_prototype(): prototype=my_prototype, ) got = await a.getitem(selection=(slice(0, 10), slice(0, 10)), prototype=my_prototype) - assert isinstance(got, TestNDArrayLike) - assert np.array_equal(expect, got) + # ignoring a mypy error here that TestNDArrayLike doesn't meet the NDArrayLike protocol + # The test passes, so it clearly does. + assert isinstance(got, TestNDArrayLike) # type: ignore[unreachable] + assert np.array_equal(expect, got) # type: ignore[unreachable] @gpu_test @pytest.mark.asyncio -async def test_codecs_use_of_gpu_prototype(): +async def test_codecs_use_of_gpu_prototype() -> None: expect = cp.zeros((10, 10), dtype="uint16", order="F") a = await AsyncArray.create( StorePath(MemoryStore(mode="w")) / "test_codecs_use_of_gpu_prototype", @@ -147,7 +153,7 @@ async def test_codecs_use_of_gpu_prototype(): assert cp.array_equal(expect, got) -def test_numpy_buffer_prototype(): +def test_numpy_buffer_prototype() -> None: buffer = cpu.buffer_prototype.buffer.create_zero_length() ndbuffer = cpu.buffer_prototype.nd_buffer.create(shape=(1, 2), dtype=np.dtype("int64")) assert isinstance(buffer.as_array_like(), np.ndarray) From 59b118d3d0620af917afe1c82c9d2ede45711e46 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Fri, 6 Sep 2024 20:47:20 +0200 Subject: [PATCH 0141/1043] Fix new ruff/flake8-type-checking TCH003 error (#2154) TCH003 Move standard library import into a type-checking block --- tests/v3/test_buffer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 06c0714333..5a313dc1ab 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -1,6 +1,6 @@ from __future__ import annotations -import types +from typing import TYPE_CHECKING import numpy as np import pytest @@ -23,6 +23,9 @@ ) from zarr.testing.utils import gpu_test +if TYPE_CHECKING: + import types + try: import cupy as cp except ImportError: From 8e29f3744db8bdf9434b0ee104207734b6257b59 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sun, 8 Sep 2024 10:59:43 +0100 Subject: [PATCH 0142/1043] Autogenerate all API docs (#2002) * Autogenerate all API docs * Fix duplicate Array docs * Fix doc errors * Fix skip logic * Fix module name splitting * Change autoapi directory * Add more enum docs * Fix testing docstring * Pin max version of sphinx-autoapi * Fix spelling * Pin max version of sphinx-autoapi * Skip zarr.core in API docs * Fix buffer namespace --- .gitignore | 1 + docs/Makefile | 1 + docs/api/index.rst | 2 +- docs/api/zarr.rst | 5 ----- docs/conf.py | 18 +++++++++++++++--- pyproject.toml | 4 ++-- src/zarr/codecs/blosc.py | 8 ++++++++ src/zarr/codecs/bytes.py | 4 ++++ src/zarr/codecs/sharding.py | 4 ++++ src/zarr/core/array.py | 3 +++ src/zarr/core/buffer/core.py | 3 +++ src/zarr/core/indexing.py | 6 +++++- src/zarr/testing/strategies.py | 9 ++++++--- src/zarr/v2/indexing.py | 2 +- src/zarr/v2/meta.py | 3 +-- src/zarr/v2/n5.py | 1 + 16 files changed, 56 insertions(+), 18 deletions(-) delete mode 100644 docs/api/zarr.rst diff --git a/.gitignore b/.gitignore index 84bcb00ffb..a09fb54d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ coverage.xml # Sphinx documentation docs/_build/ +docs/_autoapi # PyBuilder target/ diff --git a/docs/Makefile b/docs/Makefile index e6adc1ca8c..fc8fa12915 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -52,6 +52,7 @@ help: .PHONY: clean clean: rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR)/../_autoapi .PHONY: html html: diff --git a/docs/api/index.rst b/docs/api/index.rst index 58ea531905..8735180cd9 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -4,4 +4,4 @@ API Reference .. toctree:: :maxdepth: 1 - zarr + ../_autoapi/zarr/index diff --git a/docs/api/zarr.rst b/docs/api/zarr.rst deleted file mode 100644 index 8a9216f19e..0000000000 --- a/docs/api/zarr.rst +++ /dev/null @@ -1,5 +0,0 @@ -zarr -==== - -.. autoapimodule:: zarr - :members: diff --git a/docs/conf.py b/docs/conf.py index 35afa60577..c2805d6239 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,9 +57,10 @@ autoapi_dirs = ['../src/zarr'] autoapi_add_toctree_entry = False -autoapi_generate_api_docs = False +autoapi_generate_api_docs = True autoapi_member_order = "groupwise" -autoapi_root = "api" +autoapi_root = "_autoapi" +autoapi_keep_files = True # Add any paths that contain templates here, relative to this directory. @@ -172,8 +173,19 @@ html_logo = "_static/logo1.png" +def autoapi_skip_modules(app: sphinx.application.Sphinx, what: str, name: str, obj: object, skip: bool, options: dict[str, Any]) -> bool: + """ + Return True if a module should be skipped in th API docs. + """ + parts = name.split(".") + if what == "module" and (any(part.startswith("_") for part in parts) or "v2" in name or name.startswith("zarr.core")): + return True + return False + + def setup(app: sphinx.application.Sphinx) -> None: app.add_css_file("custom.css") + app.connect("autoapi-skip-member", autoapi_skip_modules) # The name of an image file (relative to this directory) to use as a favicon of @@ -339,7 +351,7 @@ def setup(app: sphinx.application.Sphinx) -> None: # use in refs e.g: # :ref:`comparison manual ` intersphinx_mapping = { - "python": ("https://docs.python.org/", None), + "python": ("https://docs.python.org/3/", None), "numpy": ("https://numpy.org/doc/stable/", None), "numcodecs": ("https://numcodecs.readthedocs.io/en/stable/", None), } diff --git a/pyproject.toml b/pyproject.toml index c6ab4a4638..da0763d093 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,9 +78,9 @@ gpu = [ "cupy-cuda12x", ] docs = [ - 'sphinx', + 'sphinx<8', 'sphinx-autobuild>=2021.3.14', - 'sphinx-autoapi', + 'sphinx-autoapi<3.1', 'sphinx_design', 'sphinx-issues', 'sphinx-copybutton', diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index cd265e3dc0..f831dc960d 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -21,6 +21,10 @@ class BloscShuffle(Enum): + """ + Enum for shuffle filter used by blosc. + """ + noshuffle = "noshuffle" shuffle = "shuffle" bitshuffle = "bitshuffle" @@ -38,6 +42,10 @@ def from_int(cls, num: int) -> BloscShuffle: class BloscCname(Enum): + """ + Enum for compression library used by blosc. + """ + lz4 = "lz4" lz4hc = "lz4hc" blosclz = "blosclz" diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index 4561b22346..bc3207be2e 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -19,6 +19,10 @@ class Endian(Enum): + """ + Enum for endian type used by bytes codec. + """ + big = "big" little = "little" diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 9eb8e2ffb3..5281104724 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -60,6 +60,10 @@ class ShardingCodecIndexLocation(Enum): + """ + Enum for index location used by the sharding codec. + """ + start = "start" end = "end" diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index fbe0b19f6b..dcd7217d7d 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -68,6 +68,9 @@ from zarr.abc.codec import Codec, CodecPipeline +# Array and AsyncArray are defined in the base ``zarr`` namespace +__all__ = ["parse_array_metadata", "create_codec_pipeline"] + def parse_array_metadata(data: Any) -> ArrayV2Metadata | ArrayV3Metadata: if isinstance(data, ArrayV2Metadata | ArrayV3Metadata): diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index fca2da2b77..6aff47e8d3 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -29,6 +29,9 @@ from zarr.codecs.bytes import Endian from zarr.core.common import BytesLike, ChunkCoords +# Everything here is imported into ``zarr.core.buffer`` namespace. +__all__: list[str] = [] + @runtime_checkable class ArrayLike(Protocol): diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index e6e04e58f6..caf57e212a 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -559,6 +559,10 @@ def __iter__(self) -> Iterator[ChunkDimProjection]: class Order(Enum): + """ + Enum for indexing order. + """ + UNKNOWN = 0 INCREASING = 1 DECREASING = 2 @@ -700,7 +704,7 @@ def slice_to_range(s: slice, length: int) -> range: def ix_(selection: Any, shape: ChunkCoords) -> npt.NDArray[np.intp]: - """Convert an orthogonal selection to a numpy advanced (fancy) selection, like numpy.ix_ + """Convert an orthogonal selection to a numpy advanced (fancy) selection, like ``numpy.ix_`` but with support for slices and single ints.""" # normalisation diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 3a460d4fff..83de3d92ce 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -155,9 +155,12 @@ def basic_indices(draw: st.DrawFn, *, shape: tuple[int], **kwargs): # type: ign def key_ranges(keys: SearchStrategy = node_names) -> SearchStrategy[list]: - """fn to generate key_ranges strategy for get_partial_values() - returns list strategy w/ form: [(key, (range_start, range_step)), - (key, (range_start, range_step)),...] + """ + Function to generate key_ranges strategy for get_partial_values() + returns list strategy w/ form:: + + [(key, (range_start, range_step)), + (key, (range_start, range_step)),...] """ byte_ranges = st.tuples( st.none() | st.integers(min_value=0), st.none() | st.integers(min_value=0) diff --git a/src/zarr/v2/indexing.py b/src/zarr/v2/indexing.py index bb2d9f1adb..880baf3f72 100644 --- a/src/zarr/v2/indexing.py +++ b/src/zarr/v2/indexing.py @@ -534,7 +534,7 @@ def slice_to_range(s: slice, l: int): # noqa: E741 def ix_(selection, shape): - """Convert an orthogonal selection to a numpy advanced (fancy) selection, like numpy.ix_ + """Convert an orthogonal selection to a numpy advanced (fancy) selection, like ``numpy.ix_`` but with support for slices and single ints.""" # normalisation diff --git a/src/zarr/v2/meta.py b/src/zarr/v2/meta.py index 418c0727cf..2f7ce1242e 100644 --- a/src/zarr/v2/meta.py +++ b/src/zarr/v2/meta.py @@ -17,8 +17,7 @@ # FLOAT_FILLS = {"NaN": np.nan, "Infinity": np.PINF, "-Infinity": np.NINF} -_v3_core_types = set("".join(d) for d in itertools.product("<>", ("u", "i", "f"), ("2", "4", "8"))) -_v3_core_types = {"bool", "i1", "u1"} | _v3_core_types +_v3_core_types = {"bool", "i1", "u1"} | set("".join(d) for d in itertools.product("<>", ("u", "i", "f"), ("2", "4", "8"))) # The set of complex types allowed ({"c8", ">c16"}) _v3_complex_types = set(f"{end}c{_bytes}" for end, _bytes in itertools.product("<>", ("8", "16"))) diff --git a/src/zarr/v2/n5.py b/src/zarr/v2/n5.py index a6fd39f5b8..ece110f49d 100644 --- a/src/zarr/v2/n5.py +++ b/src/zarr/v2/n5.py @@ -272,6 +272,7 @@ class N5FSStore(FSStore): """Implementation of the N5 format (https://github.com/saalfeldlab/n5) using `fsspec`, which allows storage on a variety of filesystems. Based on `zarr.N5Store`. + Parameters ---------- path : string From aa9da2b63e6d7ba35774f232b1e66be0575a8a3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:34:50 +0100 Subject: [PATCH 0143/1043] Bump pypa/gh-action-pypi-publish from 1.10.0 to 1.10.1 in the actions group (#2161) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.0 to 1.10.1 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.0...v1.10.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 9f115ba3e6..2d7cbd49b6 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.0 + - uses: pypa/gh-action-pypi-publish@v1.10.1 with: user: __token__ password: ${{ secrets.pypi_password }} From fb9c5261109b637dfca636aebf50d839cbf3eae4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:35:38 +0100 Subject: [PATCH 0144/1043] Bump pypa/gh-action-pypi-publish from 1.10.0 to 1.10.1 in the actions group (#2160) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.0 to 1.10.1 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.0...v1.10.1) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 1783afd806..bcff0f6154 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -55,7 +55,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.0 + - uses: pypa/gh-action-pypi-publish@v1.10.1 with: user: __token__ password: ${{ secrets.pypi_password }} From ac6c6a3cf88976ab296c94f0891fee2ac7ae1bdb Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 9 Sep 2024 13:03:39 -0500 Subject: [PATCH 0145/1043] Ensure that store_dict used for empty dicts (#2162) * Ensure that store_dict used for empty dicts The `store_dict or {}` is Falsey for empty dicts, so the user provided dict wasn't being used. * fixup --------- Co-authored-by: David Stansby --- src/zarr/store/memory.py | 4 +++- tests/v3/test_store/test_memory.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index e0e72b1827..4f2627bf67 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -30,7 +30,9 @@ def __init__( mode: AccessModeLiteral = "r", ): super().__init__(mode=mode) - self._store_dict = store_dict or {} + if store_dict is None: + store_dict = {} + self._store_dict = store_dict async def empty(self) -> bool: return not self._store_dict diff --git a/tests/v3/test_store/test_memory.py b/tests/v3/test_store/test_memory.py index f76423c631..e64f93ad95 100644 --- a/tests/v3/test_store/test_memory.py +++ b/tests/v3/test_store/test_memory.py @@ -18,11 +18,14 @@ def set(self, store: MemoryStore, key: str, value: Buffer) -> None: def get(self, store: MemoryStore, key: str) -> Buffer: return store._store_dict[key] - @pytest.fixture(scope="function", params=[None, {}]) + @pytest.fixture(scope="function", params=[None, True]) def store_kwargs( self, request: pytest.FixtureRequest ) -> dict[str, str | None | dict[str, Buffer]]: - return {"store_dict": request.param, "mode": "r+"} + kwargs = {"store_dict": None, "mode": "r+"} + if request.param is True: + kwargs["store_dict"] = {} + return kwargs @pytest.fixture(scope="function") def store(self, store_kwargs: str | None | dict[str, Buffer]) -> MemoryStore: @@ -77,3 +80,9 @@ def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: def test_list_prefix(self, store: GpuMemoryStore) -> None: assert True + + +def test_uses_dict(): + store_dict = {} + store = MemoryStore(store_dict) + assert store._store_dict is store_dict From 774f5b1e5a84e7cea6b6dfb913c4330854e712b8 Mon Sep 17 00:00:00 2001 From: Jonny Saunders Date: Tue, 10 Sep 2024 07:09:08 -0700 Subject: [PATCH 0146/1043] [docs] remove primary sidebar from tutorial (#2142) * remove primary sidebar from tutorial * lint --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 886160382f..136fcf32d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,7 +195,7 @@ def setup(app): # html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -# html_sidebars = {} +html_sidebars = {"tutorial": []} # Additional templates that should be rendered to pages, maps page names to # template names. From c2d50a6b199d1f469864b462528425a3a64b795c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:44:19 -0700 Subject: [PATCH 0147/1043] chore: update pre-commit hooks (#2165) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update pre-commit hooks updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.3 → v0.6.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.3...v0.6.4) - [github.com/scientific-python/cookie: 2024.04.23 → 2024.08.19](https://github.com/scientific-python/cookie/compare/2024.04.23...2024.08.19) * remove src setting from ruff config --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Joseph Hamman --- .pre-commit-config.yaml | 4 ++-- pyproject.toml | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11ec156d20..1f7373c028 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.3 + rev: v0.6.4 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -41,7 +41,7 @@ repos: # Zarr v2 - types-redis - repo: https://github.com/scientific-python/cookie - rev: 2024.04.23 + rev: 2024.08.19 hooks: - id: sp-repo-review - repo: https://github.com/pre-commit/pygrep-hooks diff --git a/pyproject.toml b/pyproject.toml index da0763d093..d92b89c65e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -181,7 +181,6 @@ serve = "sphinx-autobuild docs docs/_build --host 0.0.0.0" [tool.ruff] line-length = 100 -src = ["src"] force-exclude = true extend-exclude = [ ".bzr", From 84034388cc37b35df08c18c1979dbe849e0b288b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 11 Sep 2024 01:24:25 -0500 Subject: [PATCH 0148/1043] Fixed test warnings (#2168) --- src/zarr/testing/buffer.py | 4 ++++ tests/v3/test_group.py | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/zarr/testing/buffer.py b/src/zarr/testing/buffer.py index 119ffc0a66..9d640d2c64 100644 --- a/src/zarr/testing/buffer.py +++ b/src/zarr/testing/buffer.py @@ -25,10 +25,14 @@ class TestNDArrayLike(np.ndarray): """An example of a ndarray-like class""" + __test__ = False + class TestBuffer(cpu.Buffer): """Example of a custom Buffer that handles ArrayLike""" + __test__ = False + class NDBufferUsingTestNDArrayLike(cpu.NDBuffer): """Example of a custom NDBuffer that handles MyNDArrayLike""" diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index e588b6a11b..6989b406da 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -785,14 +785,15 @@ async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: Zarr async def test_create_dataset(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: root = await AsyncGroup.create(store=store, zarr_format=zarr_format) - foo = await root.create_dataset("foo", shape=(10,), dtype="uint8") + with pytest.warns(DeprecationWarning): + foo = await root.create_dataset("foo", shape=(10,), dtype="uint8") assert foo.shape == (10,) - with pytest.raises(ContainsArrayError): + with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): await root.create_dataset("foo", shape=(100,), dtype="int8") _ = await root.create_group("bar") - with pytest.raises(ContainsGroupError): + with pytest.raises(ContainsGroupError), pytest.warns(DeprecationWarning): await root.create_dataset("bar", shape=(100,), dtype="int8") From 67819a11b7972480c49d266b2d46e6f1b5a08709 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Wed, 11 Sep 2024 01:24:51 -0500 Subject: [PATCH 0149/1043] Fixed path segment duplication in open_array (#2167) Closes https://github.com/zarr-developers/zarr-python/issues/2166 --- src/zarr/api/asynchronous.py | 1 - src/zarr/testing/store.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 7f59517f39..abd5a6acb9 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -897,7 +897,6 @@ async def open_array( if store_path.store.mode.create: return await create( store=store_path, - path=path, zarr_format=zarr_format, overwrite=store_path.store.mode.overwrite, **kwargs, diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 925087ae67..65d7751a0d 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -2,6 +2,7 @@ import pytest +import zarr.api.asynchronous from zarr.abc.store import AccessMode, Store from zarr.core.buffer import Buffer, default_buffer_prototype from zarr.store._utils import _normalize_interval_index @@ -232,3 +233,14 @@ async def test_list_dir(self, store: S) -> None: keys_observed = [k async for k in store.list_dir("group-0/group-1")] assert len(keys_expected) == len(keys_observed), keys_observed assert set(keys_observed) == set(keys_expected), keys_observed + + async def test_set_get(self, store_kwargs: dict[str, Any]) -> None: + kwargs = {**store_kwargs, **{"mode": "w"}} + store = self.store_cls(**kwargs) + await zarr.api.asynchronous.open_array(store=store, path="a", mode="w", shape=(4,)) + keys = [x async for x in store.list()] + assert keys == ["a/zarr.json"] + + # no errors + await zarr.api.asynchronous.open_array(store=store, path="a", mode="r") + await zarr.api.asynchronous.open_array(store=store, path="a", mode="a") From 726fdfbf569c144310893440a40ee8ee05e6524e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Wed, 11 Sep 2024 07:25:11 +0100 Subject: [PATCH 0150/1043] Simplify mypy config for tests (#2156) * Simplify mypy config for tests * Add a list of test modules to expliclty ignore typing * More test mypy fixes * Remove old test_metadata file * Fix ignoring v2 tests * Fix name test --- .pre-commit-config.yaml | 2 +- pyproject.toml | 19 ++++++++++++++++- tests/{v3/test_metadata.py => __init__.py} | 0 tests/v3/conftest.py | 24 +++++++++++----------- tests/v3/test_codec_entrypoints.py | 9 ++++---- tests/v3/test_common.py | 2 +- tests/v3/test_config.py | 17 +++++++-------- tests/v3/test_group.py | 2 +- tests/v3/test_indexing.py | 15 +++++++------- tests/v3/test_metadata/__init__.py | 0 tests/v3/test_metadata/test_v3.py | 2 +- tests/v3/test_properties.py | 6 +++--- tests/v3/test_store/__init__.py | 0 tests/v3/test_store/test_core.py | 4 ++-- tests/v3/test_store/test_remote.py | 10 +++++---- tests/v3/test_store/test_stateful_store.py | 4 ++-- tests/v3/test_sync.py | 11 +++++----- tests/v3/test_v2.py | 5 ++--- 18 files changed, 75 insertions(+), 57 deletions(-) rename tests/{v3/test_metadata.py => __init__.py} (100%) create mode 100644 tests/v3/test_metadata/__init__.py create mode 100644 tests/v3/test_store/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1f7373c028..2fd405b64d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: rev: v1.11.2 hooks: - id: mypy - files: src|tests/v3/test_(api|array|buffer).py + files: src|tests additional_dependencies: # Package dependencies - asciitree diff --git a/pyproject.toml b/pyproject.toml index d92b89c65e..49878b61f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -225,6 +225,7 @@ python_version = "3.10" ignore_missing_imports = true namespace_packages = false + strict = true warn_unreachable = true @@ -236,6 +237,22 @@ module = [ ] ignore_errors = true +[[tool.mypy.overrides]] +module = [ + "tests.v2.*", + "tests.v3.package_with_entrypoint.*", + "tests.v3.test_codecs.*", + "tests.v3.test_metadata.*", + "tests.v3.test_store.*", + "tests.v3.test_config", + "tests.v3.test_group", + "tests.v3.test_indexing", + "tests.v3.test_properties", + "tests.v3.test_sync", + "tests.v3.test_v2", +] +ignore_errors = true + [tool.pytest.ini_options] minversion = "7" testpaths = ["tests"] @@ -262,6 +279,6 @@ markers = [ [tool.repo-review] ignore = [ - "PC111", # fix Python code in documentation - enable later + "PC111", # fix Python code in documentation - enable later "PC180", # for JavaScript - not interested ] diff --git a/tests/v3/test_metadata.py b/tests/__init__.py similarity index 100% rename from tests/v3/test_metadata.py rename to tests/__init__.py diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index fcb5e3c867..b1308f058f 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -1,11 +1,20 @@ from __future__ import annotations +import pathlib +from dataclasses import dataclass, field from typing import TYPE_CHECKING +import numpy as np +import numpy.typing as npt +import pytest +from hypothesis import HealthCheck, Verbosity, settings + from zarr import AsyncGroup, config +from zarr.store import LocalStore, MemoryStore, StorePath +from zarr.store.remote import RemoteStore if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import Generator, Iterator from types import ModuleType from typing import Any, Literal @@ -13,15 +22,6 @@ from zarr.abc.store import Store from zarr.core.common import ChunkCoords, MemoryOrder, ZarrFormat -import pathlib -from dataclasses import dataclass, field - -import numpy as np -import pytest -from hypothesis import HealthCheck, Verbosity, settings - -from zarr.store import LocalStore, MemoryStore, StorePath -from zarr.store.remote import RemoteStore async def parse_store( @@ -102,7 +102,7 @@ def xp(request: pytest.FixtureRequest) -> Iterator[ModuleType]: @pytest.fixture(autouse=True) -def reset_config(): +def reset_config() -> Generator[None, None, None]: config.reset() yield config.reset() @@ -116,7 +116,7 @@ class ArrayRequest: @pytest.fixture -def array_fixture(request: pytest.FixtureRequest) -> np.ndarray: +def array_fixture(request: pytest.FixtureRequest) -> npt.NDArray[Any]: array_request: ArrayRequest = request.param return ( np.arange(np.prod(array_request.shape)) diff --git a/tests/v3/test_codec_entrypoints.py b/tests/v3/test_codec_entrypoints.py index 9e2932fdd5..95dae68762 100644 --- a/tests/v3/test_codec_entrypoints.py +++ b/tests/v3/test_codec_entrypoints.py @@ -1,5 +1,6 @@ import os.path import sys +from collections.abc import Generator import pytest @@ -10,7 +11,7 @@ @pytest.fixture() -def set_path(): +def set_path() -> Generator[None, None, None]: sys.path.append(here) zarr.registry._collect_entrypoints() yield @@ -23,14 +24,14 @@ def set_path(): @pytest.mark.usefixtures("set_path") @pytest.mark.parametrize("codec_name", ["TestEntrypointCodec", "TestEntrypointGroup.Codec"]) -def test_entrypoint_codec(codec_name): +def test_entrypoint_codec(codec_name: str) -> None: config.set({"codecs.test": "package_with_entrypoint." + codec_name}) cls_test = zarr.registry.get_codec_class("test") assert cls_test.__qualname__ == codec_name @pytest.mark.usefixtures("set_path") -def test_entrypoint_pipeline(): +def test_entrypoint_pipeline() -> None: config.set({"codec_pipeline.path": "package_with_entrypoint.TestEntrypointCodecPipeline"}) cls = zarr.registry.get_pipeline_class() assert cls.__name__ == "TestEntrypointCodecPipeline" @@ -38,7 +39,7 @@ def test_entrypoint_pipeline(): @pytest.mark.usefixtures("set_path") @pytest.mark.parametrize("buffer_name", ["TestEntrypointBuffer", "TestEntrypointGroup.Buffer"]) -def test_entrypoint_buffer(buffer_name): +def test_entrypoint_buffer(buffer_name: str) -> None: config.set( { "buffer": "package_with_entrypoint." + buffer_name, diff --git a/tests/v3/test_common.py b/tests/v3/test_common.py index 85c6d93d36..c28723d1a8 100644 --- a/tests/v3/test_common.py +++ b/tests/v3/test_common.py @@ -109,7 +109,7 @@ def test_parse_shapelike_valid(data: Iterable[int]) -> None: # todo: more dtypes @pytest.mark.parametrize("data", [("uint8", np.uint8), ("float64", np.float64)]) -def parse_dtype(data: tuple[str, np.dtype]) -> None: +def parse_dtype(data: tuple[str, np.dtype[Any]]) -> None: unparsed, parsed = data assert parse_dtype(unparsed) == parsed diff --git a/tests/v3/test_config.py b/tests/v3/test_config.py index b3e04e51da..c0674ecbfd 100644 --- a/tests/v3/test_config.py +++ b/tests/v3/test_config.py @@ -10,7 +10,7 @@ import zarr from zarr import Array, zeros from zarr.abc.codec import CodecInput, CodecOutput, CodecPipeline -from zarr.abc.store import ByteSetter +from zarr.abc.store import ByteSetter, Store from zarr.codecs import BatchedCodecPipeline, BloscCodec, BytesCodec, Crc32cCodec, ShardingCodec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer @@ -77,17 +77,18 @@ def test_config_defaults_can_be_overridden(key: str, old_val: Any, new_val: Any) assert config.get(key) == new_val -def test_fully_qualified_name(): +def test_fully_qualified_name() -> None: class MockClass: pass - assert "v3.test_config.test_fully_qualified_name..MockClass" == fully_qualified_name( - MockClass + assert ( + fully_qualified_name(MockClass) + == "tests.v3.test_config.test_fully_qualified_name..MockClass" ) @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) -def test_config_codec_pipeline_class(store): +def test_config_codec_pipeline_class(store: Store) -> None: # has default value assert get_pipeline_class().__name__ != "" @@ -138,7 +139,7 @@ class MockEnvCodecPipeline(CodecPipeline): @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) -def test_config_codec_implementation(store): +def test_config_codec_implementation(store: Store) -> None: # has default value assert fully_qualified_name(get_codec_class("blosc")) == config.defaults[0]["codecs"]["blosc"] @@ -171,7 +172,7 @@ async def _encode_single( @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) -def test_config_ndbuffer_implementation(store): +def test_config_ndbuffer_implementation(store: Store) -> None: # has default value assert fully_qualified_name(get_ndbuffer_class()) == config.defaults[0]["ndbuffer"] @@ -191,7 +192,7 @@ def test_config_ndbuffer_implementation(store): assert isinstance(got, TestNDArrayLike) -def test_config_buffer_implementation(): +def test_config_buffer_implementation() -> None: # has default value assert fully_qualified_name(get_buffer_class()) == config.defaults[0]["buffer"] diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 6989b406da..64d4cbc867 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -681,7 +681,7 @@ async def test_asyncgroup_update_attributes( assert agroup_new_attributes.attrs == attributes_new -async def test_group_members_async(store: LocalStore | MemoryStore): +async def test_group_members_async(store: LocalStore | MemoryStore) -> None: group = AsyncGroup( GroupMetadata(), store_path=StorePath(store=store, path="root"), diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index 3230bbc675..0b08378726 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -25,12 +25,11 @@ if TYPE_CHECKING: from collections.abc import Iterator - from zarr.abc.store import Store from zarr.core.common import ChunkCoords @pytest.fixture -async def store() -> Iterator[Store]: +async def store() -> Iterator[StorePath]: yield StorePath(await MemoryStore.open(mode="w")) @@ -52,7 +51,7 @@ def zarr_array_from_numpy_array( class CountingDict(MemoryStore): @classmethod - async def open(cls): + async def open(cls) -> CountingDict: store = await super().open(mode="w") store.counter = Counter() return store @@ -68,7 +67,7 @@ async def set(self, key, value, byte_range=None): return await super().set(key, value, byte_range) -def test_normalize_integer_selection(): +def test_normalize_integer_selection() -> None: assert 1 == normalize_integer_selection(1, 100) assert 99 == normalize_integer_selection(-1, 100) with pytest.raises(IndexError): @@ -79,7 +78,7 @@ def test_normalize_integer_selection(): normalize_integer_selection(-1000, 100) -def test_replace_ellipsis(): +def test_replace_ellipsis() -> None: # 1D, single item assert (0,) == replace_ellipsis(0, (100,)) @@ -258,7 +257,7 @@ def _test_get_basic_selection(a, z, selection): # noinspection PyStatementEffect -def test_get_basic_selection_1d(store: StorePath): +def test_get_basic_selection_1d(store: StorePath) -> None: # setup a = np.arange(1050, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) @@ -328,7 +327,7 @@ def test_get_basic_selection_1d(store: StorePath): # noinspection PyStatementEffect -def test_get_basic_selection_2d(store: StorePath): +def test_get_basic_selection_2d(store: StorePath) -> None: # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) @@ -349,7 +348,7 @@ def test_get_basic_selection_2d(store: StorePath): np.testing.assert_array_equal(z[fancy_selection], [0, 11]) -def test_fancy_indexing_fallback_on_get_setitem(store: StorePath): +def test_fancy_indexing_fallback_on_get_setitem(store: StorePath) -> None: z = zarr_array_from_numpy_array(store, np.zeros((20, 20))) z[[1, 2, 3], [1, 2, 3]] = 1 np.testing.assert_array_equal( diff --git a/tests/v3/test_metadata/__init__.py b/tests/v3/test_metadata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v3/test_metadata/test_v3.py b/tests/v3/test_metadata/test_v3.py index bc43154a54..5e5d22f0b1 100644 --- a/tests/v3/test_metadata/test_v3.py +++ b/tests/v3/test_metadata/test_v3.py @@ -237,7 +237,7 @@ def test_metadata_to_dict( @pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) @pytest.mark.parametrize("precision", ["ns", "D"]) -async def test_datetime_metadata(fill_value: int, precision: str): +async def test_datetime_metadata(fill_value: int, precision: str) -> None: metadata_dict = { "zarr_format": 3, "node_type": "array", diff --git a/tests/v3/test_properties.py b/tests/v3/test_properties.py index c978187cf1..7a085c03b7 100644 --- a/tests/v3/test_properties.py +++ b/tests/v3/test_properties.py @@ -11,7 +11,7 @@ @given(st.data()) -def test_roundtrip(data): +def test_roundtrip(data: st.DataObject) -> None: nparray = data.draw(np_arrays) zarray = data.draw(arrays(arrays=st.just(nparray))) assert_array_equal(nparray, zarray[:]) @@ -23,7 +23,7 @@ def test_roundtrip(data): # Uncomment the next line to reproduce the original failure. # @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/ndR2z7nkDZEDADWpBL4=') @pytest.mark.filterwarnings("ignore::RuntimeWarning") -def test_basic_indexing(data): +def test_basic_indexing(data: st.DataObject) -> None: zarray = data.draw(arrays()) nparray = zarray[:] indexer = data.draw(basic_indices(shape=nparray.shape)) @@ -42,7 +42,7 @@ def test_basic_indexing(data): # Uncomment the next line to reproduce the original failure. # @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/eLmF7qr/C5EDADZUBRM=') @pytest.mark.filterwarnings("ignore::RuntimeWarning") -def test_vindex(data): +def test_vindex(data: st.DataObject) -> None: zarray = data.draw(arrays()) nparray = zarray[:] diff --git a/tests/v3/test_store/__init__.py b/tests/v3/test_store/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v3/test_store/test_core.py b/tests/v3/test_store/test_core.py index 23821acfa6..c65d91f9d0 100644 --- a/tests/v3/test_store/test_core.py +++ b/tests/v3/test_store/test_core.py @@ -7,7 +7,7 @@ from zarr.store.memory import MemoryStore -async def test_make_store_path(tmpdir) -> None: +async def test_make_store_path(tmpdir: str) -> None: # None store_path = await make_store_path(None) assert isinstance(store_path.store, MemoryStore) @@ -33,4 +33,4 @@ async def test_make_store_path(tmpdir) -> None: assert Path(store_path.store.root) == Path(tmpdir) with pytest.raises(TypeError): - await make_store_path(1) + await make_store_path(1) # type: ignore[arg-type] diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index b09bf24e24..7495bec8e1 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -1,5 +1,7 @@ import os +from collections.abc import Generator +import botocore.client import fsspec import pytest from upath import UPath @@ -22,7 +24,7 @@ @pytest.fixture(scope="module") -def s3_base(): +def s3_base() -> Generator[None, None, None]: # writable local S3 system # This fixture is module-scoped, meaning that we can reuse the MotoServer across all tests @@ -37,7 +39,7 @@ def s3_base(): server.stop() -def get_boto3_client(): +def get_boto3_client() -> botocore.client.BaseClient: from botocore.session import Session # NB: we use the sync botocore client for setup @@ -46,7 +48,7 @@ def get_boto3_client(): @pytest.fixture(autouse=True, scope="function") -def s3(s3_base): +def s3(s3_base) -> Generator[s3fs.S3FileSystem, None, None]: """ Quoting Martin Durant: pytest-asyncio creates a new event loop for each async test. @@ -81,7 +83,7 @@ async def alist(it): return out -async def test_basic(): +async def test_basic() -> None: store = await RemoteStore.open( f"s3://{test_bucket_name}", mode="w", endpoint_url=endpoint_url, anon=False ) diff --git a/tests/v3/test_store/test_stateful_store.py b/tests/v3/test_store/test_stateful_store.py index a8f51e96e6..d062f9235f 100644 --- a/tests/v3/test_store/test_stateful_store.py +++ b/tests/v3/test_store/test_stateful_store.py @@ -102,7 +102,7 @@ class ZarrStoreStateMachine(RuleBasedStateMachine): https://hypothesis.readthedocs.io/en/latest/stateful.html """ - def __init__(self): + def __init__(self) -> None: super().__init__() self.model: dict[str, bytes] = {} self.store = SyncStoreWrapper(MemoryStore(mode="w")) @@ -170,7 +170,7 @@ def delete(self, data: bytes) -> None: del self.model[key] @rule() - def clear(self): + def clear(self) -> None: assert not self.store.mode.readonly note("(clear)") self.store.clear() diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index a335f9b48c..22834747e7 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -8,11 +8,10 @@ @pytest.fixture(params=[True, False]) -def sync_loop(request) -> asyncio.AbstractEventLoop | None: +def sync_loop(request: pytest.FixtureRequest) -> asyncio.AbstractEventLoop | None: if request.param is True: return _get_loop() - - if request.param is False: + else: return None @@ -58,7 +57,7 @@ def foo() -> str: return "foo" with pytest.raises(TypeError): - sync(foo(), loop=sync_loop) + sync(foo(), loop=sync_loop) # type: ignore[arg-type] @pytest.mark.filterwarnings("ignore:coroutine.*was never awaited") @@ -82,7 +81,7 @@ def foo() -> str: return "foo" async def bar() -> str: - return sync(foo(), loop=sync_loop) + return sync(foo(), loop=sync_loop) # type: ignore[arg-type] with pytest.raises(SyncError): sync(bar(), loop=sync_loop) @@ -92,7 +91,7 @@ async def bar() -> str: def test_sync_raises_if_loop_is_invalid_type() -> None: foo = AsyncMock(return_value="foo") with pytest.raises(TypeError): - sync(foo(), loop=1) + sync(foo(), loop=1) # type: ignore[arg-type] foo.assert_not_awaited() diff --git a/tests/v3/test_v2.py b/tests/v3/test_v2.py index 4f4dc5aed3..9ddde68e23 100644 --- a/tests/v3/test_v2.py +++ b/tests/v3/test_v2.py @@ -4,16 +4,15 @@ import pytest from zarr import Array -from zarr.abc.store import Store from zarr.store import MemoryStore, StorePath @pytest.fixture -async def store() -> Iterator[Store]: +async def store() -> Iterator[StorePath]: yield StorePath(await MemoryStore.open(mode="w")) -def test_simple(store: Store): +def test_simple(store: StorePath) -> None: data = np.arange(0, 256, dtype="uint16").reshape((16, 16)) a = Array.create( From e1d98cdf3d35e4cb2815912d82a64c1623347c87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 23:35:06 -0700 Subject: [PATCH 0151/1043] Bump pytest from 8.3.2 to 8.3.3 in the requirements group (#2172) Bumps the requirements group with 1 update: [pytest](https://github.com/pytest-dev/pytest). Updates `pytest` from 8.3.2 to 8.3.3 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.2...8.3.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements_dev_minimal.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements_dev_minimal.txt b/requirements_dev_minimal.txt index 2991d99299..1d29404d6c 100644 --- a/requirements_dev_minimal.txt +++ b/requirements_dev_minimal.txt @@ -5,4 +5,4 @@ numcodecs==0.13.0 msgpack-python==0.5.6 setuptools-scm==8.1.0 # test requirements -pytest==8.3.2 +pytest==8.3.3 From c62294e37effa31be98c88d99ac6f1ec4ea5ae46 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 12 Sep 2024 12:32:57 -0500 Subject: [PATCH 0152/1043] Accept dictionaries for `store` argument (#2164) * Accept dictionaries for `store` * fixup doc build warnings * Only support dictionaries --- src/zarr/api/asynchronous.py | 12 +++++++++++- src/zarr/core/buffer/core.py | 4 ++-- src/zarr/core/buffer/cpu.py | 4 ++-- src/zarr/core/buffer/gpu.py | 12 ++++++------ src/zarr/store/common.py | 6 +++++- tests/v3/test_group.py | 12 ++++++++++++ 6 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index abd5a6acb9..5fbdbbfb06 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -497,8 +497,18 @@ async def open_group( Parameters ---------- - store : Store or string, optional + store : Store, string, or mapping, optional Store or path to directory in file system or name of zip file. + + Strings are interpreted as paths on the local file system + and used as the ``root`` argument to :class:`zarr.store.LocalStore`. + + Dictionaries are used as the ``store_dict`` argument in + :class:`zarr.store.MemoryStore``. + + By default (``store=None``) a new :class:`zarr.store.MemoryStore` + is created. + mode : {'r', 'r+', 'a', 'w', 'w-'}, optional Persistence mode: 'r' means read only (must exist); 'r+' means read/write (must exist); 'a' means read/write (create if doesn't diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 6aff47e8d3..ff26478ca9 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -195,8 +195,8 @@ def from_buffer(cls, buffer: Buffer) -> Self: ------- A new buffer representing the content of the input buffer - Note - ---- + Notes + ----- Subclasses of `Buffer` must override this method to implement more optimal conversions that avoid copies where possible """ diff --git a/src/zarr/core/buffer/cpu.py b/src/zarr/core/buffer/cpu.py index 66d5ecfe7c..cef16209ec 100644 --- a/src/zarr/core/buffer/cpu.py +++ b/src/zarr/core/buffer/cpu.py @@ -73,8 +73,8 @@ def from_buffer(cls, buffer: core.Buffer) -> Self: ------- A new buffer representing the content of the input buffer - Note - ---- + Notes + ----- Subclasses of `Buffer` must override this method to implement more optimal conversions that avoid copies where possible """ diff --git a/src/zarr/core/buffer/gpu.py b/src/zarr/core/buffer/gpu.py index 476a4a62c1..c817431d3d 100644 --- a/src/zarr/core/buffer/gpu.py +++ b/src/zarr/core/buffer/gpu.py @@ -38,8 +38,8 @@ class Buffer(core.Buffer): array-like instance can be copied/converted to a regular Numpy array (host memory). - Note - ---- + Notes + ----- This buffer is untyped, so all indexing and sizes are in bytes. Parameters @@ -123,8 +123,8 @@ class NDBuffer(core.NDBuffer): ndarray-like instance can be copied/converted to a regular Numpy array (host memory). - Note - ---- + Notes + ----- The two buffer classes Buffer and NDBuffer are very similar. In fact, Buffer is a special case of NDBuffer where dim=1, stride=1, and dtype="b". However, in order to use Python's type system to differentiate between the contiguous @@ -193,8 +193,8 @@ def from_numpy_array(cls, array_like: npt.ArrayLike) -> Self: def as_numpy_array(self) -> npt.NDArray[Any]: """Returns the buffer as a NumPy array (host memory). - Warning - ------- + Warnings + -------- Might have to copy data, consider using `.as_ndarray_like()` instead. Returns diff --git a/src/zarr/store/common.py b/src/zarr/store/common.py index 6a88de7760..8028c9af3d 100644 --- a/src/zarr/store/common.py +++ b/src/zarr/store/common.py @@ -71,7 +71,7 @@ def __eq__(self, other: Any) -> bool: return False -StoreLike = Store | StorePath | Path | str +StoreLike = Store | StorePath | Path | str | dict[str, Buffer] async def make_store_path( @@ -94,6 +94,10 @@ async def make_store_path( return StorePath(await LocalStore.open(root=store_like, mode=mode or "r")) elif isinstance(store_like, str): return StorePath(await LocalStore.open(root=Path(store_like), mode=mode or "r")) + elif isinstance(store_like, dict): + # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. + # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. + return StorePath(await MemoryStore.open(store_dict=store_like, mode=mode)) raise TypeError diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 64d4cbc867..de09d1b0c1 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -5,7 +5,9 @@ import numpy as np import pytest +import zarr.api.asynchronous from zarr import Array, AsyncArray, AsyncGroup, Group +from zarr.api.synchronous import open_group from zarr.core.buffer import default_buffer_prototype from zarr.core.common import ZarrFormat from zarr.core.group import GroupMetadata @@ -820,3 +822,13 @@ async def test_require_array(store: LocalStore | MemoryStore, zarr_format: ZarrF _ = await root.create_group("bar") with pytest.raises(TypeError, match="Incompatible object"): await root.require_array("bar", shape=(10,), dtype="int8") + + +async def test_open_mutable_mapping(): + group = await zarr.api.asynchronous.open_group(store={}, mode="w") + assert isinstance(group.store_path.store, MemoryStore) + + +def test_open_mutable_mapping_sync(): + group = open_group(store={}, mode="w") + assert isinstance(group.store_path.store, MemoryStore) From 52d68490b308ad861ed9d128b320628ce690afea Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 12 Sep 2024 20:30:27 +0200 Subject: [PATCH 0153/1043] refactor: split metadata into v2 and v3 modules (#2163) * refactor: split metadata into v2 and v3 modules * add more explicit typeguards * port fill value normalization from v2 * remove v2 suffix from zarr format parsing * remove v2 suffix from zarr format parsing --- src/zarr/api/asynchronous.py | 3 +- src/zarr/codecs/sharding.py | 2 +- src/zarr/core/array.py | 4 +- src/zarr/core/metadata/__init__.py | 4 + src/zarr/core/metadata/common.py | 67 +++ src/zarr/core/metadata/v2.py | 235 ++++++++ src/zarr/core/{metadata.py => metadata/v3.py} | 505 +++++------------- tests/v3/test_metadata/test_v2.py | 8 +- tests/v3/test_metadata/test_v3.py | 5 +- 9 files changed, 448 insertions(+), 385 deletions(-) create mode 100644 src/zarr/core/metadata/__init__.py create mode 100644 src/zarr/core/metadata/common.py create mode 100644 src/zarr/core/metadata/v2.py rename src/zarr/core/{metadata.py => metadata/v3.py} (61%) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 5fbdbbfb06..d757aa2120 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -10,7 +10,8 @@ from zarr.core.array import Array, AsyncArray from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat from zarr.core.group import AsyncGroup -from zarr.core.metadata import ArrayV2Metadata, ArrayV3Metadata +from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.store import ( StoreLike, make_store_path, diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index 5281104724..df7f5978a7 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -44,7 +44,7 @@ get_indexer, morton_order_iter, ) -from zarr.core.metadata import parse_codecs +from zarr.core.metadata.v3 import parse_codecs from zarr.registry import get_ndbuffer_class, get_pipeline_class, register_codec if TYPE_CHECKING: diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index dcd7217d7d..7311b6eec2 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -55,7 +55,8 @@ is_scalar, pop_fields, ) -from zarr.core.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata +from zarr.core.metadata.v2 import ArrayV2Metadata +from zarr.core.metadata.v3 import ArrayV3Metadata from zarr.core.sync import sync from zarr.registry import get_pipeline_class from zarr.store import StoreLike, StorePath, make_store_path @@ -67,6 +68,7 @@ from collections.abc import Iterable from zarr.abc.codec import Codec, CodecPipeline + from zarr.core.metadata.common import ArrayMetadata # Array and AsyncArray are defined in the base ``zarr`` namespace __all__ = ["parse_array_metadata", "create_codec_pipeline"] diff --git a/src/zarr/core/metadata/__init__.py b/src/zarr/core/metadata/__init__.py new file mode 100644 index 0000000000..addf473397 --- /dev/null +++ b/src/zarr/core/metadata/__init__.py @@ -0,0 +1,4 @@ +from .v2 import ArrayV2Metadata +from .v3 import ArrayV3Metadata + +__all__ = ["ArrayV2Metadata", "ArrayV3Metadata"] diff --git a/src/zarr/core/metadata/common.py b/src/zarr/core/metadata/common.py new file mode 100644 index 0000000000..583375b4b7 --- /dev/null +++ b/src/zarr/core/metadata/common.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Literal + + import numpy as np + from typing_extensions import Self + + from zarr.core.array_spec import ArraySpec + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.common import JSON, ChunkCoords, ZarrFormat + +from abc import ABC, abstractmethod +from dataclasses import dataclass + +from zarr.abc.metadata import Metadata + + +@dataclass(frozen=True, kw_only=True) +class ArrayMetadata(Metadata, ABC): + shape: ChunkCoords + fill_value: Any + chunk_grid: ChunkGrid + attributes: dict[str, JSON] + zarr_format: ZarrFormat + + @property + @abstractmethod + def dtype(self) -> np.dtype[Any]: + pass + + @property + @abstractmethod + def ndim(self) -> int: + pass + + @abstractmethod + def get_chunk_spec( + self, _chunk_coords: ChunkCoords, order: Literal["C", "F"], prototype: BufferPrototype + ) -> ArraySpec: + pass + + @abstractmethod + def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: + pass + + @abstractmethod + def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: + pass + + @abstractmethod + def update_shape(self, shape: ChunkCoords) -> Self: + pass + + @abstractmethod + def update_attributes(self, attributes: dict[str, JSON]) -> Self: + pass + + +def parse_attributes(data: None | dict[str, JSON]) -> dict[str, JSON]: + if data is None: + return {} + + return data diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py new file mode 100644 index 0000000000..6d5ecd7e86 --- /dev/null +++ b/src/zarr/core/metadata/v2.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Literal + + import numpy.typing as npt + from typing_extensions import Self + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.common import JSON, ChunkCoords + +import json +from dataclasses import dataclass, field, replace + +import numpy as np + +from zarr.core.array_spec import ArraySpec +from zarr.core.chunk_grids import RegularChunkGrid +from zarr.core.chunk_key_encodings import parse_separator +from zarr.core.common import ZARRAY_JSON, ZATTRS_JSON, parse_dtype, parse_shapelike +from zarr.core.config import config, parse_indexing_order +from zarr.core.metadata.common import ArrayMetadata, parse_attributes + + +@dataclass(frozen=True, kw_only=True) +class ArrayV2Metadata(ArrayMetadata): + shape: ChunkCoords + chunk_grid: RegularChunkGrid + data_type: np.dtype[Any] + fill_value: None | int | float = 0 + order: Literal["C", "F"] = "C" + filters: list[dict[str, JSON]] | None = None + dimension_separator: Literal[".", "/"] = "." + compressor: dict[str, JSON] | None = None + attributes: dict[str, JSON] = field(default_factory=dict) + zarr_format: Literal[2] = field(init=False, default=2) + + def __init__( + self, + *, + shape: ChunkCoords, + dtype: npt.DTypeLike, + chunks: ChunkCoords, + fill_value: Any, + order: Literal["C", "F"], + dimension_separator: Literal[".", "/"] = ".", + compressor: dict[str, JSON] | None = None, + filters: list[dict[str, JSON]] | None = None, + attributes: dict[str, JSON] | None = None, + ): + """ + Metadata for a Zarr version 2 array. + """ + shape_parsed = parse_shapelike(shape) + data_type_parsed = parse_dtype(dtype) + chunks_parsed = parse_shapelike(chunks) + compressor_parsed = parse_compressor(compressor) + order_parsed = parse_indexing_order(order) + dimension_separator_parsed = parse_separator(dimension_separator) + filters_parsed = parse_filters(filters) + fill_value_parsed = parse_fill_value(fill_value, dtype=data_type_parsed) + attributes_parsed = parse_attributes(attributes) + + object.__setattr__(self, "shape", shape_parsed) + object.__setattr__(self, "data_type", data_type_parsed) + object.__setattr__(self, "chunk_grid", RegularChunkGrid(chunk_shape=chunks_parsed)) + object.__setattr__(self, "compressor", compressor_parsed) + object.__setattr__(self, "order", order_parsed) + object.__setattr__(self, "dimension_separator", dimension_separator_parsed) + object.__setattr__(self, "filters", filters_parsed) + object.__setattr__(self, "fill_value", fill_value_parsed) + object.__setattr__(self, "attributes", attributes_parsed) + + # ensure that the metadata document is consistent + _ = parse_metadata(self) + + @property + def ndim(self) -> int: + return len(self.shape) + + @property + def dtype(self) -> np.dtype[Any]: + return self.data_type + + @property + def chunks(self) -> ChunkCoords: + return self.chunk_grid.chunk_shape + + def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: + def _json_convert( + o: Any, + ) -> Any: + if isinstance(o, np.dtype): + if o.fields is None: + return o.str + else: + return o.descr + if np.isscalar(o): + # convert numpy scalar to python type, and pass + # python types through + return getattr(o, "item", lambda: o)() + raise TypeError + + zarray_dict = self.to_dict() + + # todo: remove this check when we can ensure that to_dict always returns dicts. + if not isinstance(zarray_dict, dict): + raise TypeError(f"Invalid type: got {type(zarray_dict)}, expected dict.") + + zattrs_dict = zarray_dict.pop("attributes", {}) + json_indent = config.get("json_indent") + return { + ZARRAY_JSON: prototype.buffer.from_bytes( + json.dumps(zarray_dict, default=_json_convert, indent=json_indent).encode() + ), + ZATTRS_JSON: prototype.buffer.from_bytes( + json.dumps(zattrs_dict, indent=json_indent).encode() + ), + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: + # make a copy to protect the original from modification + _data = data.copy() + # check that the zarr_format attribute is correct + _ = parse_zarr_format(_data.pop("zarr_format")) + return cls(**_data) + + def to_dict(self) -> JSON: + zarray_dict = super().to_dict() + + # todo: remove this check when we can ensure that to_dict always returns dicts. + if not isinstance(zarray_dict, dict): + raise TypeError(f"Invalid type: got {type(zarray_dict)}, expected dict.") + + _ = zarray_dict.pop("chunk_grid") + zarray_dict["chunks"] = self.chunk_grid.chunk_shape + + _ = zarray_dict.pop("data_type") + zarray_dict["dtype"] = self.data_type.str + + return zarray_dict + + def get_chunk_spec( + self, _chunk_coords: ChunkCoords, order: Literal["C", "F"], prototype: BufferPrototype + ) -> ArraySpec: + return ArraySpec( + shape=self.chunk_grid.chunk_shape, + dtype=self.dtype, + fill_value=self.fill_value, + order=order, + prototype=prototype, + ) + + def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: + chunk_identifier = self.dimension_separator.join(map(str, chunk_coords)) + return "0" if chunk_identifier == "" else chunk_identifier + + def update_shape(self, shape: ChunkCoords) -> Self: + return replace(self, shape=shape) + + def update_attributes(self, attributes: dict[str, JSON]) -> Self: + return replace(self, attributes=attributes) + + +def parse_zarr_format(data: Literal[2]) -> Literal[2]: + if data == 2: + return data + raise ValueError(f"Invalid value. Expected 2. Got {data}.") + + +def parse_filters(data: list[dict[str, JSON]] | None) -> list[dict[str, JSON]] | None: + return data + + +def parse_compressor(data: dict[str, JSON] | None) -> dict[str, JSON] | None: + return data + + +def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: + if (l_chunks := len(data.chunks)) != (l_shape := len(data.shape)): + msg = ( + f"The `shape` and `chunks` attributes must have the same length. " + f"`chunks` has length {l_chunks}, but `shape` has length {l_shape}." + ) + raise ValueError(msg) + return data + + +def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: + """ + Parse a potential fill value into a value that is compatible with the provided dtype. + + Parameters + ---------- + fill_value: Any + A potential fill value. + dtype: np.dtype[Any] + A numpy dtype. + + Returns + An instance of `dtype`, or `None`, or any python object (in the case of an object dtype) + """ + + if fill_value is None or dtype.hasobject: + # no fill value + pass + elif not isinstance(fill_value, np.void) and fill_value == 0: + # this should be compatible across numpy versions for any array type, including + # structured arrays + fill_value = np.zeros((), dtype=dtype)[()] + + elif dtype.kind == "U": + # special case unicode because of encoding issues on Windows if passed through numpy + # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713 + + if not isinstance(fill_value, str): + raise ValueError( + f"fill_value {fill_value!r} is not valid for dtype {dtype}; must be a unicode string" + ) + else: + try: + if isinstance(fill_value, bytes) and dtype.kind == "V": + # special case for numpy 1.14 compatibility + fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()] + else: + fill_value = np.array(fill_value, dtype=dtype)[()] + + except Exception as e: + msg = f"Fill_value {fill_value} is not valid for dtype {dtype}." + raise ValueError(msg) from e + + return fill_value diff --git a/src/zarr/core/metadata.py b/src/zarr/core/metadata/v3.py similarity index 61% rename from src/zarr/core/metadata.py rename to src/zarr/core/metadata/v3.py index d25559cd53..195c3bd0a2 100644 --- a/src/zarr/core/metadata.py +++ b/src/zarr/core/metadata/v3.py @@ -1,156 +1,73 @@ from __future__ import annotations +from typing import TYPE_CHECKING, cast, overload + +if TYPE_CHECKING: + import numpy.typing as npt + from typing_extensions import Self + + from zarr.core.buffer import Buffer, BufferPrototype + from zarr.core.chunk_grids import ChunkGrid + from zarr.core.common import JSON, ChunkCoords + import json -from abc import ABC, abstractmethod from collections.abc import Iterable, Sequence from dataclasses import dataclass, field, replace from enum import Enum -from typing import TYPE_CHECKING, Any, Literal, cast, overload +from typing import Any, Literal +import numcodecs.abc import numpy as np -import numpy.typing as npt from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec, CodecPipeline -from zarr.abc.metadata import Metadata -from zarr.core.buffer import Buffer, BufferPrototype, default_buffer_prototype +from zarr.core.array_spec import ArraySpec +from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid -from zarr.core.chunk_key_encodings import ChunkKeyEncoding, parse_separator +from zarr.core.chunk_key_encodings import ChunkKeyEncoding +from zarr.core.common import ZARR_JSON, parse_dtype, parse_named_configuration, parse_shapelike +from zarr.core.config import config +from zarr.core.metadata.common import ArrayMetadata, parse_attributes from zarr.registry import get_codec_class, get_pipeline_class -if TYPE_CHECKING: - from typing_extensions import Self - -import numcodecs.abc - -from zarr.core.array_spec import ArraySpec -from zarr.core.common import ( - JSON, - ZARR_JSON, - ZARRAY_JSON, - ZATTRS_JSON, - ChunkCoords, - ZarrFormat, - parse_dtype, - parse_named_configuration, - parse_shapelike, -) -from zarr.core.config import config, parse_indexing_order - -# For type checking -_bool = bool -__all__ = ["ArrayMetadata"] - - -class DataType(Enum): - bool = "bool" - int8 = "int8" - int16 = "int16" - int32 = "int32" - int64 = "int64" - uint8 = "uint8" - uint16 = "uint16" - uint32 = "uint32" - uint64 = "uint64" - float32 = "float32" - float64 = "float64" - - @property - def byte_count(self) -> int: - data_type_byte_counts = { - DataType.bool: 1, - DataType.int8: 1, - DataType.int16: 2, - DataType.int32: 4, - DataType.int64: 8, - DataType.uint8: 1, - DataType.uint16: 2, - DataType.uint32: 4, - DataType.uint64: 8, - DataType.float32: 4, - DataType.float64: 8, - } - return data_type_byte_counts[self] - - @property - def has_endianness(self) -> _bool: - # This might change in the future, e.g. for a complex with 2 8-bit floats - return self.byte_count != 1 - - def to_numpy_shortname(self) -> str: - data_type_to_numpy = { - DataType.bool: "bool", - DataType.int8: "i1", - DataType.int16: "i2", - DataType.int32: "i4", - DataType.int64: "i8", - DataType.uint8: "u1", - DataType.uint16: "u2", - DataType.uint32: "u4", - DataType.uint64: "u8", - DataType.float32: "f4", - DataType.float64: "f8", - } - return data_type_to_numpy[self] - - @classmethod - def from_dtype(cls, dtype: np.dtype[Any]) -> DataType: - dtype_to_data_type = { - "|b1": "bool", - "bool": "bool", - "|i1": "int8", - " Literal[3]: + if data == 3: + return data + raise ValueError(f"Invalid value. Expected 3. Got {data}.") -@dataclass(frozen=True, kw_only=True) -class ArrayMetadata(Metadata, ABC): - shape: ChunkCoords - fill_value: Any - chunk_grid: ChunkGrid - attributes: dict[str, JSON] - zarr_format: ZarrFormat +def parse_node_type_array(data: Literal["array"]) -> Literal["array"]: + if data == "array": + return data + raise ValueError(f"Invalid value. Expected 'array'. Got {data}.") - @property - @abstractmethod - def dtype(self) -> np.dtype[Any]: - pass - @property - @abstractmethod - def ndim(self) -> int: - pass +def parse_codecs(data: Iterable[Codec | dict[str, JSON]]) -> tuple[Codec, ...]: + out: tuple[Codec, ...] = () - @abstractmethod - def get_chunk_spec( - self, _chunk_coords: ChunkCoords, order: Literal["C", "F"], prototype: BufferPrototype - ) -> ArraySpec: - pass + if not isinstance(data, Iterable): + raise TypeError(f"Expected iterable, got {type(data)}") - @abstractmethod - def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: - pass + for c in data: + if isinstance( + c, ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec + ): # Can't use Codec here because of mypy limitation + out += (c,) + else: + name_parsed, _ = parse_named_configuration(c, require_configuration=False) + out += (get_codec_class(name_parsed).from_dict(c),) - @abstractmethod - def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - pass + return out - @abstractmethod - def update_shape(self, shape: ChunkCoords) -> Self: - pass - @abstractmethod - def update_attributes(self, attributes: dict[str, JSON]) -> Self: - pass +def parse_dimension_names(data: None | Iterable[str | None]) -> tuple[str | None, ...] | None: + if data is None: + return data + elif all(isinstance(x, type(None) | str) for x in data): + return tuple(data) + else: + msg = f"Expected either None or a iterable of str, got {type(data)}" + raise TypeError(msg) @dataclass(frozen=True, kw_only=True) @@ -186,7 +103,7 @@ def __init__( chunk_grid_parsed = ChunkGrid.from_dict(chunk_grid) chunk_key_encoding_parsed = ChunkKeyEncoding.from_dict(chunk_key_encoding) dimension_names_parsed = parse_dimension_names(dimension_names) - fill_value_parsed = parse_fill_value_v3(fill_value, dtype=data_type_parsed) + fill_value_parsed = parse_fill_value(fill_value, dtype=data_type_parsed) attributes_parsed = parse_attributes(attributes) codecs_parsed_partial = parse_codecs(codecs) @@ -294,7 +211,7 @@ def from_dict(cls, data: dict[str, JSON]) -> ArrayV3Metadata: _data = data.copy() # TODO: Remove the type: ignores[] comments below and use a TypedDict to type `data` # check that the zarr_format attribute is correct - _ = parse_zarr_format_v3(_data.pop("zarr_format")) # type: ignore[arg-type] + _ = parse_zarr_format(_data.pop("zarr_format")) # type: ignore[arg-type] # check that the node_type attribute is correct _ = parse_node_type_array(_data.pop("node_type")) # type: ignore[arg-type] @@ -323,250 +240,14 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) -@dataclass(frozen=True, kw_only=True) -class ArrayV2Metadata(ArrayMetadata): - shape: ChunkCoords - chunk_grid: RegularChunkGrid - data_type: np.dtype[Any] - fill_value: None | int | float = 0 - order: Literal["C", "F"] = "C" - filters: list[dict[str, JSON]] | None = None - dimension_separator: Literal[".", "/"] = "." - compressor: dict[str, JSON] | None = None - attributes: dict[str, JSON] = field(default_factory=dict) - zarr_format: Literal[2] = field(init=False, default=2) - - def __init__( - self, - *, - shape: ChunkCoords, - dtype: npt.DTypeLike, - chunks: ChunkCoords, - fill_value: Any, - order: Literal["C", "F"], - dimension_separator: Literal[".", "/"] = ".", - compressor: dict[str, JSON] | None = None, - filters: list[dict[str, JSON]] | None = None, - attributes: dict[str, JSON] | None = None, - ): - """ - Metadata for a Zarr version 2 array. - """ - shape_parsed = parse_shapelike(shape) - data_type_parsed = parse_dtype(dtype) - chunks_parsed = parse_shapelike(chunks) - compressor_parsed = parse_compressor(compressor) - order_parsed = parse_indexing_order(order) - dimension_separator_parsed = parse_separator(dimension_separator) - filters_parsed = parse_filters(filters) - fill_value_parsed = parse_fill_value_v2(fill_value, dtype=data_type_parsed) - attributes_parsed = parse_attributes(attributes) - - object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "data_type", data_type_parsed) - object.__setattr__(self, "chunk_grid", RegularChunkGrid(chunk_shape=chunks_parsed)) - object.__setattr__(self, "compressor", compressor_parsed) - object.__setattr__(self, "order", order_parsed) - object.__setattr__(self, "dimension_separator", dimension_separator_parsed) - object.__setattr__(self, "filters", filters_parsed) - object.__setattr__(self, "fill_value", fill_value_parsed) - object.__setattr__(self, "attributes", attributes_parsed) - - # ensure that the metadata document is consistent - _ = parse_v2_metadata(self) - - @property - def ndim(self) -> int: - return len(self.shape) - - @property - def dtype(self) -> np.dtype[Any]: - return self.data_type - - @property - def chunks(self) -> ChunkCoords: - return self.chunk_grid.chunk_shape - - def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - def _json_convert( - o: Any, - ) -> Any: - if isinstance(o, np.dtype): - if o.fields is None: - return o.str - else: - return o.descr - if np.isscalar(o): - # convert numpy scalar to python type, and pass - # python types through - return getattr(o, "item", lambda: o)() - raise TypeError - - zarray_dict = self.to_dict() - assert isinstance(zarray_dict, dict) - zattrs_dict = zarray_dict.pop("attributes", {}) - assert isinstance(zattrs_dict, dict) - json_indent = config.get("json_indent") - return { - ZARRAY_JSON: prototype.buffer.from_bytes( - json.dumps(zarray_dict, default=_json_convert, indent=json_indent).encode() - ), - ZATTRS_JSON: prototype.buffer.from_bytes( - json.dumps(zattrs_dict, indent=json_indent).encode() - ), - } - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: - # make a copy to protect the original from modification - _data = data.copy() - # check that the zarr_format attribute is correct - _ = parse_zarr_format_v2(_data.pop("zarr_format")) - return cls(**_data) - - def to_dict(self) -> JSON: - zarray_dict = super().to_dict() - - assert isinstance(zarray_dict, dict) - - _ = zarray_dict.pop("chunk_grid") - zarray_dict["chunks"] = self.chunk_grid.chunk_shape - - _ = zarray_dict.pop("data_type") - zarray_dict["dtype"] = self.data_type.str - - return zarray_dict - - def get_chunk_spec( - self, _chunk_coords: ChunkCoords, order: Literal["C", "F"], prototype: BufferPrototype - ) -> ArraySpec: - return ArraySpec( - shape=self.chunk_grid.chunk_shape, - dtype=self.dtype, - fill_value=self.fill_value, - order=order, - prototype=prototype, - ) - - def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: - chunk_identifier = self.dimension_separator.join(map(str, chunk_coords)) - return "0" if chunk_identifier == "" else chunk_identifier - - def update_shape(self, shape: ChunkCoords) -> Self: - return replace(self, shape=shape) - - def update_attributes(self, attributes: dict[str, JSON]) -> Self: - return replace(self, attributes=attributes) - - -def parse_dimension_names(data: None | Iterable[str | None]) -> tuple[str | None, ...] | None: - if data is None: - return data - elif all(isinstance(x, type(None) | str) for x in data): - return tuple(data) - else: - msg = f"Expected either None or a iterable of str, got {type(data)}" - raise TypeError(msg) - - -# todo: real validation -def parse_attributes(data: None | dict[str, JSON]) -> dict[str, JSON]: - if data is None: - return {} - - return data - - -# todo: move to its own module and drop _v3 suffix -# todo: consider folding all the literal parsing into a single function -# that takes 2 arguments -def parse_zarr_format_v3(data: Literal[3]) -> Literal[3]: - if data == 3: - return data - raise ValueError(f"Invalid value. Expected 3. Got {data}.") - - -# todo: move to its own module and drop _v2 suffix -def parse_zarr_format_v2(data: Literal[2]) -> Literal[2]: - if data == 2: - return data - raise ValueError(f"Invalid value. Expected 2. Got {data}.") - - -def parse_node_type_array(data: Literal["array"]) -> Literal["array"]: - if data == "array": - return data - raise ValueError(f"Invalid value. Expected 'array'. Got {data}.") - - -# todo: real validation -def parse_filters(data: list[dict[str, JSON]] | None) -> list[dict[str, JSON]] | None: - return data - - -# todo: real validation -def parse_compressor(data: dict[str, JSON] | None) -> dict[str, JSON] | None: - return data - - -def parse_v2_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: - if (l_chunks := len(data.chunks)) != (l_shape := len(data.shape)): - msg = ( - f"The `shape` and `chunks` attributes must have the same length. " - f"`chunks` has length {l_chunks}, but `shape` has length {l_shape}." - ) - raise ValueError(msg) - return data - - def create_pipeline(data: Iterable[Codec | JSON]) -> CodecPipeline: if not isinstance(data, Iterable): raise TypeError(f"Expected iterable, got {type(data)}") return get_pipeline_class().from_dict(data) -def parse_codecs(data: Iterable[Codec | dict[str, JSON]]) -> tuple[Codec, ...]: - out: tuple[Codec, ...] = () - - if not isinstance(data, Iterable): - raise TypeError(f"Expected iterable, got {type(data)}") - - for c in data: - if isinstance( - c, ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec - ): # Can't use Codec here because of mypy limitation - out += (c,) - else: - name_parsed, _ = parse_named_configuration(c, require_configuration=False) - out += (get_codec_class(name_parsed).from_dict(c),) - - return out - - -def parse_fill_value_v2(fill_value: Any, dtype: np.dtype[Any]) -> Any: - """ - Parse a potential fill value into a value that is compatible with the provided dtype. - - This is a light wrapper around zarr.v2.util.normalize_fill_value. - - Parameters - ---------- - fill_value: Any - A potential fill value. - dtype: np.dtype[Any] - A numpy dtype. - - Returns - An instance of `dtype`, or `None`, or any python object (in the case of an object dtype) - """ - from zarr.v2.util import normalize_fill_value - - return normalize_fill_value(fill_value=fill_value, dtype=dtype) - - BOOL = np.bool_ BOOL_DTYPE = np.dtypes.BoolDType - INTEGER_DTYPE = ( np.dtypes.Int8DType | np.dtypes.Int16DType @@ -577,33 +258,31 @@ def parse_fill_value_v2(fill_value: Any, dtype: np.dtype[Any]) -> Any: | np.dtypes.UInt32DType | np.dtypes.UInt64DType ) - INTEGER = np.int8 | np.int16 | np.int32 | np.int64 | np.uint8 | np.uint16 | np.uint32 | np.uint64 FLOAT_DTYPE = np.dtypes.Float16DType | np.dtypes.Float32DType | np.dtypes.Float64DType FLOAT = np.float16 | np.float32 | np.float64 COMPLEX_DTYPE = np.dtypes.Complex64DType | np.dtypes.Complex128DType COMPLEX = np.complex64 | np.complex128 -# todo: r* dtypes @overload -def parse_fill_value_v3(fill_value: Any, dtype: BOOL_DTYPE) -> BOOL: ... +def parse_fill_value(fill_value: Any, dtype: BOOL_DTYPE) -> BOOL: ... @overload -def parse_fill_value_v3(fill_value: Any, dtype: INTEGER_DTYPE) -> INTEGER: ... +def parse_fill_value(fill_value: Any, dtype: INTEGER_DTYPE) -> INTEGER: ... @overload -def parse_fill_value_v3(fill_value: Any, dtype: FLOAT_DTYPE) -> FLOAT: ... +def parse_fill_value(fill_value: Any, dtype: FLOAT_DTYPE) -> FLOAT: ... @overload -def parse_fill_value_v3(fill_value: Any, dtype: COMPLEX_DTYPE) -> COMPLEX: ... +def parse_fill_value(fill_value: Any, dtype: COMPLEX_DTYPE) -> COMPLEX: ... @overload -def parse_fill_value_v3(fill_value: Any, dtype: np.dtype[Any]) -> Any: +def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: # This dtype[Any] is unfortunately necessary right now. # See https://github.com/zarr-developers/zarr-python/issues/2131#issuecomment-2318010899 # for more details, but `dtype` here (which comes from `parse_dtype`) @@ -614,7 +293,7 @@ def parse_fill_value_v3(fill_value: Any, dtype: np.dtype[Any]) -> Any: ... -def parse_fill_value_v3( +def parse_fill_value( fill_value: Any, dtype: BOOL_DTYPE | INTEGER_DTYPE | FLOAT_DTYPE | COMPLEX_DTYPE | np.dtype[Any] ) -> BOOL | INTEGER | FLOAT | COMPLEX | Any: """ @@ -655,3 +334,77 @@ def parse_fill_value_v3( msg = f"Cannot parse non-string sequence {fill_value} as a scalar with type {dtype}." raise TypeError(msg) return dtype.type(fill_value) + + +# For type checking +_bool = bool + + +class DataType(Enum): + bool = "bool" + int8 = "int8" + int16 = "int16" + int32 = "int32" + int64 = "int64" + uint8 = "uint8" + uint16 = "uint16" + uint32 = "uint32" + uint64 = "uint64" + float32 = "float32" + float64 = "float64" + + @property + def byte_count(self) -> int: + data_type_byte_counts = { + DataType.bool: 1, + DataType.int8: 1, + DataType.int16: 2, + DataType.int32: 4, + DataType.int64: 8, + DataType.uint8: 1, + DataType.uint16: 2, + DataType.uint32: 4, + DataType.uint64: 8, + DataType.float32: 4, + DataType.float64: 8, + } + return data_type_byte_counts[self] + + @property + def has_endianness(self) -> _bool: + # This might change in the future, e.g. for a complex with 2 8-bit floats + return self.byte_count != 1 + + def to_numpy_shortname(self) -> str: + data_type_to_numpy = { + DataType.bool: "bool", + DataType.int8: "i1", + DataType.int16: "i2", + DataType.int32: "i4", + DataType.int64: "i8", + DataType.uint8: "u1", + DataType.uint16: "u2", + DataType.uint32: "u4", + DataType.uint64: "u8", + DataType.float32: "f4", + DataType.float64: "f8", + } + return data_type_to_numpy[self] + + @classmethod + def from_dtype(cls, dtype: np.dtype[Any]) -> DataType: + dtype_to_data_type = { + "|b1": "bool", + "bool": "bool", + "|i1": "int8", + " None: - assert parse_zarr_format_v2(2) == 2 + assert parse_zarr_format(2) == 2 @pytest.mark.parametrize("data", [None, 1, 3, 4, 5, "3"]) def test_parse_zarr_format_invalid(data: Any) -> None: with pytest.raises(ValueError, match=f"Invalid value. Expected 2. Got {data}"): - parse_zarr_format_v2(data) + parse_zarr_format(data) @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) diff --git a/tests/v3/test_metadata/test_v3.py b/tests/v3/test_metadata/test_v3.py index 5e5d22f0b1..0a545dfb9d 100644 --- a/tests/v3/test_metadata/test_v3.py +++ b/tests/v3/test_metadata/test_v3.py @@ -7,6 +7,7 @@ from zarr.codecs.bytes import BytesCodec from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_key_encodings import DefaultChunkKeyEncoding, V2ChunkKeyEncoding +from zarr.core.metadata.v3 import ArrayV3Metadata if TYPE_CHECKING: from collections.abc import Sequence @@ -18,9 +19,7 @@ import numpy as np import pytest -from zarr.core.metadata import ArrayV3Metadata, parse_dimension_names -from zarr.core.metadata import parse_fill_value_v3 as parse_fill_value -from zarr.core.metadata import parse_zarr_format_v3 as parse_zarr_format +from zarr.core.metadata.v3 import parse_dimension_names, parse_fill_value, parse_zarr_format bool_dtypes = ("bool",) From 534e0cd1291cc92445c8d304e60e728e811a3ef5 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 13 Sep 2024 15:59:18 +0100 Subject: [PATCH 0154/1043] More typing fixes for tests (#2173) --- pyproject.toml | 3 +- src/zarr/core/group.py | 2 +- tests/v3/package_with_entrypoint/__init__.py | 7 +- tests/v3/test_codecs/test_blosc.py | 14 ++- tests/v3/test_codecs/test_codecs.py | 13 +-- tests/v3/test_codecs/test_endian.py | 6 +- tests/v3/test_codecs/test_sharding.py | 15 ++- tests/v3/test_codecs/test_transpose.py | 4 +- tests/v3/test_group.py | 6 +- tests/v3/test_indexing.py | 102 ++++++++++--------- tests/v3/test_store/test_memory.py | 2 +- tests/v3/test_store/test_remote.py | 2 +- tests/v3/test_store/test_stateful_store.py | 13 +-- 13 files changed, 102 insertions(+), 87 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 49878b61f6..886cd5a0bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -241,7 +241,8 @@ ignore_errors = true module = [ "tests.v2.*", "tests.v3.package_with_entrypoint.*", - "tests.v3.test_codecs.*", + "tests.v3.test_codecs.test_codecs", + "tests.v3.test_codecs.test_transpose", "tests.v3.test_metadata.*", "tests.v3.test_store.*", "tests.v3.test_config", diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 56c9c88ea5..40815b96c8 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -516,7 +516,7 @@ async def require_array( self, name: str, *, - shape: ChunkCoords, + shape: ShapeLike, dtype: npt.DTypeLike = None, exact: bool = False, **kwargs: Any, diff --git a/tests/v3/package_with_entrypoint/__init__.py b/tests/v3/package_with_entrypoint/__init__.py index 352b9d570e..b818adf8ea 100644 --- a/tests/v3/package_with_entrypoint/__init__.py +++ b/tests/v3/package_with_entrypoint/__init__.py @@ -2,6 +2,7 @@ from numpy import ndarray +import zarr.core.buffer from zarr.abc.codec import ArrayBytesCodec, CodecInput, CodecOutput, CodecPipeline from zarr.codecs import BytesCodec from zarr.core.array_spec import ArraySpec @@ -29,7 +30,7 @@ def compute_encoded_size(self, input_byte_length: int, chunk_spec: ArraySpec) -> class TestEntrypointCodecPipeline(CodecPipeline): - def __init__(self, batch_size: int = 1): + def __init__(self, batch_size: int = 1) -> None: pass async def encode( @@ -55,10 +56,10 @@ class TestEntrypointGroup: class Codec(BytesCodec): pass - class Buffer(Buffer): + class Buffer(zarr.core.buffer.Buffer): pass - class NDBuffer(NDBuffer): + class NDBuffer(zarr.core.buffer.NDBuffer): pass class Pipeline(CodecPipeline): diff --git a/tests/v3/test_codecs/test_blosc.py b/tests/v3/test_codecs/test_blosc.py index 5de4c9fa99..4c569055b7 100644 --- a/tests/v3/test_codecs/test_blosc.py +++ b/tests/v3/test_codecs/test_blosc.py @@ -24,10 +24,9 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None: fill_value=0, codecs=[BytesCodec(), BloscCodec()], ) - - zarr_json = json.loads( - (await store.get(f"{path}/zarr.json", prototype=default_buffer_prototype())).to_bytes() - ) + buf = await store.get(f"{path}/zarr.json", prototype=default_buffer_prototype()) + assert buf is not None + zarr_json = json.loads(buf.to_bytes()) blosc_configuration_json = zarr_json["codecs"][1]["configuration"] assert blosc_configuration_json["typesize"] == typesize if typesize == 1: @@ -45,10 +44,9 @@ async def test_blosc_evolve(store: Store, dtype: str) -> None: fill_value=0, codecs=[ShardingCodec(chunk_shape=(16, 16), codecs=[BytesCodec(), BloscCodec()])], ) - - zarr_json = json.loads( - (await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype())).to_bytes() - ) + buf = await store.get(f"{path2}/zarr.json", prototype=default_buffer_prototype()) + assert buf is not None + zarr_json = json.loads(buf.to_bytes()) blosc_configuration_json = zarr_json["codecs"][0]["configuration"]["codecs"][1]["configuration"] assert blosc_configuration_json["typesize"] == typesize if typesize == 1: diff --git a/tests/v3/test_codecs/test_codecs.py b/tests/v3/test_codecs/test_codecs.py index f388fb1c1e..57103d17c2 100644 --- a/tests/v3/test_codecs/test_codecs.py +++ b/tests/v3/test_codecs/test_codecs.py @@ -7,7 +7,7 @@ import numpy as np import pytest -import zarr.v2 +import zarr.v2.creation from zarr import Array, AsyncArray, config from zarr.codecs import ( BytesCodec, @@ -23,6 +23,7 @@ if TYPE_CHECKING: from zarr.abc.codec import Codec from zarr.abc.store import Store + from zarr.core.buffer.core import NDArrayLike from zarr.core.common import MemoryOrder @@ -39,7 +40,7 @@ class _AsyncArraySelectionProxy: array: AsyncArray selection: Selection - async def get(self) -> np.ndarray: + async def get(self) -> NDArrayLike: return await self.array.getitem(self.selection) async def set(self, value: np.ndarray) -> None: @@ -119,7 +120,7 @@ async def test_order( if not with_sharding: # Compare with zarr-python - z = zarr.v2.create( + z = zarr.v2.creation.create( shape=data.shape, chunks=(32, 8), dtype=" None: fill_value=1, ) - z2 = zarr.v2.create( + z2 = zarr.v2.creation.create( shape=data.shape, chunks=(10, 10), dtype=data.dtype, @@ -310,7 +311,7 @@ async def test_zarr_compat_F(store: Store) -> None: codecs=[TransposeCodec(order=order_from_dim("F", data.ndim)), BytesCodec()], ) - z2 = zarr.v2.create( + z2 = zarr.v2.creation.create( shape=data.shape, chunks=(10, 10), dtype=data.dtype, @@ -406,7 +407,7 @@ def test_invalid_metadata(store: Store) -> None: fill_value=0, codecs=[ BytesCodec(), - TransposeCodec(order="F"), + TransposeCodec(order="F"), # type: ignore[arg-type] ], ) spath4 = StorePath(store, "invalid_missing_bytes_codec") diff --git a/tests/v3/test_codecs/test_endian.py b/tests/v3/test_codecs/test_endian.py index f97d95d9b7..3c36c90b81 100644 --- a/tests/v3/test_codecs/test_endian.py +++ b/tests/v3/test_codecs/test_endian.py @@ -3,7 +3,7 @@ import numpy as np import pytest -import zarr.v2 +import zarr.v2.creation from zarr import AsyncArray from zarr.abc.store import Store from zarr.codecs import BytesCodec @@ -35,7 +35,7 @@ async def test_endian(store: Store, endian: Literal["big", "little"]) -> None: assert np.array_equal(data, readback_data) # Compare with v2 - z = zarr.v2.create( + z = zarr.v2.creation.create( shape=data.shape, chunks=(16, 16), dtype=">u2" if endian == "big" else " None: """ Test that we can create an array with a sharding codec, write data to that array, and get @@ -80,7 +85,7 @@ def test_sharding( indirect=["array_fixture"], ) def test_sharding_partial( - store: Store, array_fixture: np.ndarray, index_location: ShardingCodecIndexLocation + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation ) -> None: data = array_fixture spath = StorePath(store) @@ -123,7 +128,7 @@ def test_sharding_partial( @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) def test_sharding_partial_read( - store: Store, array_fixture: np.ndarray, index_location: ShardingCodecIndexLocation + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation ) -> None: data = array_fixture spath = StorePath(store) @@ -160,7 +165,7 @@ def test_sharding_partial_read( @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) def test_sharding_partial_overwrite( - store: Store, array_fixture: np.ndarray, index_location: ShardingCodecIndexLocation + store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation ) -> None: data = array_fixture[:10, :10, :10] spath = StorePath(store) @@ -212,7 +217,7 @@ def test_sharding_partial_overwrite( @pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) def test_nested_sharding( store: Store, - array_fixture: np.ndarray, + array_fixture: npt.NDArray[Any], outer_index_location: ShardingCodecIndexLocation, inner_index_location: ShardingCodecIndexLocation, ) -> None: diff --git a/tests/v3/test_codecs/test_transpose.py b/tests/v3/test_codecs/test_transpose.py index a24c650c5b..c42c56034a 100644 --- a/tests/v3/test_codecs/test_transpose.py +++ b/tests/v3/test_codecs/test_transpose.py @@ -3,7 +3,7 @@ import numpy as np import pytest -import zarr.v2 +import zarr.v2.creation from zarr import Array, AsyncArray, config from zarr.abc.store import Store from zarr.codecs import BytesCodec, ShardingCodec, TransposeCodec @@ -72,7 +72,7 @@ async def test_transpose( if not with_sharding: # Compare with zarr-python - z = zarr.v2.create( + z = zarr.v2.creation.create( shape=data.shape, chunks=(1, 32, 8), dtype=" None: expect = a[selection] actual = z.get_basic_selection(selection) assert_array_equal(expect, actual) @@ -272,7 +272,7 @@ def test_get_basic_selection_1d(store: StorePath) -> None: z[selection] with pytest.raises(IndexError): - z.get_basic_selection([1, 0]) + z.get_basic_selection([1, 0]) # type: ignore[arg-type] basic_selections_2d = [ @@ -388,7 +388,9 @@ def test_fancy_indexing_fallback_on_get_setitem(store: StorePath) -> None: (([1, 0, 1]), [[3, 4, 5], [0, 1, 2], [3, 4, 5]]), ], ) -def test_orthogonal_indexing_fallback_on_getitem_2d(store: StorePath, index, expected_result): +def test_orthogonal_indexing_fallback_on_getitem_2d( + store: StorePath, index, expected_result +) -> None: """ Tests the orthogonal indexing fallback on __getitem__ for a 2D matrix. @@ -418,7 +420,9 @@ def test_orthogonal_indexing_fallback_on_getitem_2d(store: StorePath, index, exp ((slice(0, 2), [1, 2], slice(0, 2)), [[[3, 4], [6, 7]], [[12, 13], [15, 16]]]), ], ) -def test_orthogonal_indexing_fallback_on_getitem_3d(store: StorePath, index, expected_result): +def test_orthogonal_indexing_fallback_on_getitem_3d( + store: StorePath, index, expected_result +) -> None: """ Tests the orthogonal indexing fallback on __getitem__ for a 3D matrix. @@ -456,7 +460,9 @@ def test_orthogonal_indexing_fallback_on_getitem_3d(store: StorePath, index, exp (([0, 2], slice(None, None, 2)), [[1, 0, 1], [0, 0, 0], [1, 0, 1]]), ], ) -def test_orthogonal_indexing_fallback_on_setitem_2d(store: StorePath, index, expected_result): +def test_orthogonal_indexing_fallback_on_setitem_2d( + store: StorePath, index, expected_result +) -> None: """ Tests the orthogonal indexing fallback on __setitem__ for a 3D matrix. @@ -472,7 +478,7 @@ def test_orthogonal_indexing_fallback_on_setitem_2d(store: StorePath, index, exp np.testing.assert_array_equal(z[:], a, err_msg="Indexing disagrees with numpy") -def test_fancy_indexing_doesnt_mix_with_implicit_slicing(store: StorePath): +def test_fancy_indexing_doesnt_mix_with_implicit_slicing(store: StorePath) -> None: z2 = zarr_array_from_numpy_array(store, np.zeros((5, 5, 5))) with pytest.raises(IndexError): z2[[1, 2, 3], [1, 2, 3]] = 2 @@ -526,7 +532,7 @@ def test_set_basic_selection_0d( # arr_z[..., "foo", "bar"] = v[["foo", "bar"]] -def _test_get_orthogonal_selection(a, z, selection): +def _test_get_orthogonal_selection(a, z, selection) -> None: expect = oindex(a, selection) actual = z.get_orthogonal_selection(selection) assert_array_equal(expect, actual) @@ -535,7 +541,7 @@ def _test_get_orthogonal_selection(a, z, selection): # noinspection PyStatementEffect -def test_get_orthogonal_selection_1d_bool(store: StorePath): +def test_get_orthogonal_selection_1d_bool(store: StorePath) -> None: # setup a = np.arange(1050, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) @@ -556,7 +562,7 @@ def test_get_orthogonal_selection_1d_bool(store: StorePath): # noinspection PyStatementEffect -def test_get_orthogonal_selection_1d_int(store: StorePath): +def test_get_orthogonal_selection_1d_int(store: StorePath) -> None: # setup a = np.arange(1050, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) @@ -595,7 +601,7 @@ def test_get_orthogonal_selection_1d_int(store: StorePath): z.oindex[selection] -def _test_get_orthogonal_selection_2d(a, z, ix0, ix1): +def _test_get_orthogonal_selection_2d(a, z, ix0, ix1) -> None: selections = [ # index both axes with array (ix0, ix1), @@ -613,7 +619,7 @@ def _test_get_orthogonal_selection_2d(a, z, ix0, ix1): # noinspection PyStatementEffect -def test_get_orthogonal_selection_2d(store: StorePath): +def test_get_orthogonal_selection_2d(store: StorePath) -> None: # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) @@ -655,7 +661,7 @@ def test_get_orthogonal_selection_2d(store: StorePath): z.oindex[selection] -def _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2): +def _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) -> None: selections = [ # single value (84, 42, 4), @@ -690,7 +696,7 @@ def _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2): _test_get_orthogonal_selection(a, z, selection) -def test_get_orthogonal_selection_3d(store: StorePath): +def test_get_orthogonal_selection_3d(store: StorePath) -> None: # setup a = np.arange(100000, dtype=int).reshape(200, 50, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(60, 20, 3)) @@ -719,7 +725,7 @@ def test_get_orthogonal_selection_3d(store: StorePath): _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) -def test_orthogonal_indexing_edge_cases(store: StorePath): +def test_orthogonal_indexing_edge_cases(store: StorePath) -> None: a = np.arange(6).reshape(1, 2, 3) z = zarr_array_from_numpy_array(store, a, chunk_shape=(1, 2, 3)) @@ -732,7 +738,7 @@ def test_orthogonal_indexing_edge_cases(store: StorePath): assert_array_equal(expect, actual) -def _test_set_orthogonal_selection(v, a, z, selection): +def _test_set_orthogonal_selection(v, a, z, selection) -> None: for value in 42, oindex(v, selection), oindex(v, selection).tolist(): if isinstance(value, list) and value == []: # skip these cases as cannot preserve all dimensions @@ -750,7 +756,7 @@ def _test_set_orthogonal_selection(v, a, z, selection): assert_array_equal(a, z[:]) -def test_set_orthogonal_selection_1d(store: StorePath): +def test_set_orthogonal_selection_1d(store: StorePath) -> None: # setup v = np.arange(1050, dtype=int) a = np.empty(v.shape, dtype=int) @@ -776,7 +782,7 @@ def test_set_orthogonal_selection_1d(store: StorePath): _test_set_orthogonal_selection(v, a, z, selection) -def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1): +def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) -> None: selections = [ # index both axes with array (ix0, ix1), @@ -790,7 +796,7 @@ def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1): _test_set_orthogonal_selection(v, a, z, selection) -def test_set_orthogonal_selection_2d(store: StorePath): +def test_set_orthogonal_selection_2d(store: StorePath) -> None: # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -819,7 +825,7 @@ def test_set_orthogonal_selection_2d(store: StorePath): _test_set_orthogonal_selection(v, a, z, selection) -def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2): +def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) -> None: selections = ( # single value (84, 42, 4), @@ -845,7 +851,7 @@ def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2): _test_set_orthogonal_selection(v, a, z, selection) -def test_set_orthogonal_selection_3d(store: StorePath): +def test_set_orthogonal_selection_3d(store: StorePath) -> None: # setup v = np.arange(100000, dtype=int).reshape(200, 50, 10) a = np.empty_like(v) @@ -879,7 +885,7 @@ def test_set_orthogonal_selection_3d(store: StorePath): _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) -def test_orthogonal_indexing_fallback_on_get_setitem(store: StorePath): +def test_orthogonal_indexing_fallback_on_get_setitem(store: StorePath) -> None: z = zarr_array_from_numpy_array(store, np.zeros((20, 20))) z[[1, 2, 3], [1, 2, 3]] = 1 np.testing.assert_array_equal( @@ -900,7 +906,7 @@ def test_orthogonal_indexing_fallback_on_get_setitem(store: StorePath): np.testing.assert_array_equal(z2[:], [0, 1, 1, 1, 0]) -def _test_get_coordinate_selection(a, z, selection): +def _test_get_coordinate_selection(a, z, selection) -> None: expect = a[selection] actual = z.get_coordinate_selection(selection) assert_array_equal(expect, actual) @@ -924,7 +930,7 @@ def _test_get_coordinate_selection(a, z, selection): # noinspection PyStatementEffect -def test_get_coordinate_selection_1d(store: StorePath): +def test_get_coordinate_selection_1d(store: StorePath) -> None: # setup a = np.arange(1050, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) @@ -966,7 +972,7 @@ def test_get_coordinate_selection_1d(store: StorePath): z.vindex[selection] -def test_get_coordinate_selection_2d(store: StorePath): +def test_get_coordinate_selection_2d(store: StorePath) -> None: # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) @@ -1020,7 +1026,7 @@ def test_get_coordinate_selection_2d(store: StorePath): z.get_coordinate_selection(selection) -def _test_set_coordinate_selection(v, a, z, selection): +def _test_set_coordinate_selection(v, a, z, selection) -> None: for value in 42, v[selection], v[selection].tolist(): # setup expectation a[:] = 0 @@ -1035,7 +1041,7 @@ def _test_set_coordinate_selection(v, a, z, selection): assert_array_equal(a, z[:]) -def test_set_coordinate_selection_1d(store: StorePath): +def test_set_coordinate_selection_1d(store: StorePath) -> None: # setup v = np.arange(1050, dtype=int) a = np.empty(v.shape, dtype=v.dtype) @@ -1059,7 +1065,7 @@ def test_set_coordinate_selection_1d(store: StorePath): z.vindex[selection] = 42 -def test_set_coordinate_selection_2d(store: StorePath): +def test_set_coordinate_selection_2d(store: StorePath) -> None: # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -1090,7 +1096,7 @@ def test_set_coordinate_selection_2d(store: StorePath): _test_set_coordinate_selection(v, a, z, (ix0, ix1)) -def _test_get_block_selection(a, z, selection, expected_idx): +def _test_get_block_selection(a, z, selection, expected_idx) -> None: expect = a[expected_idx] actual = z.get_block_selection(selection) assert_array_equal(expect, actual) @@ -1142,7 +1148,7 @@ def _test_get_block_selection(a, z, selection, expected_idx): ] -def test_get_block_selection_1d(store: StorePath): +def test_get_block_selection_1d(store: StorePath) -> None: # setup a = np.arange(1050, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) @@ -1195,7 +1201,7 @@ def test_get_block_selection_1d(store: StorePath): ] -def test_get_block_selection_2d(store: StorePath): +def test_get_block_selection_2d(store: StorePath) -> None: # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) @@ -1216,7 +1222,9 @@ def test_get_block_selection_2d(store: StorePath): z.get_block_selection(selection) -def _test_set_block_selection(v: np.ndarray, a: np.ndarray, z: zarr.Array, selection, expected_idx): +def _test_set_block_selection( + v: np.ndarray, a: np.ndarray, z: zarr.Array, selection, expected_idx +) -> None: for value in 42, v[expected_idx], v[expected_idx].tolist(): # setup expectation a[:] = 0 @@ -1231,7 +1239,7 @@ def _test_set_block_selection(v: np.ndarray, a: np.ndarray, z: zarr.Array, selec assert_array_equal(a, z[:]) -def test_set_block_selection_1d(store: StorePath): +def test_set_block_selection_1d(store: StorePath) -> None: # setup v = np.arange(1050, dtype=int) a = np.empty(v.shape, dtype=v.dtype) @@ -1249,7 +1257,7 @@ def test_set_block_selection_1d(store: StorePath): z.blocks[selection] = 42 -def test_set_block_selection_2d(store: StorePath): +def test_set_block_selection_2d(store: StorePath) -> None: # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty(v.shape, dtype=v.dtype) @@ -1271,7 +1279,7 @@ def test_set_block_selection_2d(store: StorePath): z.set_block_selection(selection, 42) -def _test_get_mask_selection(a, z, selection): +def _test_get_mask_selection(a, z, selection) -> None: expect = a[selection] actual = z.get_mask_selection(selection) assert_array_equal(expect, actual) @@ -1297,7 +1305,7 @@ def _test_get_mask_selection(a, z, selection): # noinspection PyStatementEffect -def test_get_mask_selection_1d(store: StorePath): +def test_get_mask_selection_1d(store: StorePath) -> None: # setup a = np.arange(1050, dtype=int) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) @@ -1322,7 +1330,7 @@ def test_get_mask_selection_1d(store: StorePath): # noinspection PyStatementEffect -def test_get_mask_selection_2d(store: StorePath): +def test_get_mask_selection_2d(store: StorePath) -> None: # setup a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) @@ -1342,7 +1350,7 @@ def test_get_mask_selection_2d(store: StorePath): z.vindex[[True, False]] # wrong no. dimensions -def _test_set_mask_selection(v, a, z, selection): +def _test_set_mask_selection(v, a, z, selection) -> None: a[:] = 0 z[:] = 0 a[selection] = v[selection] @@ -1356,7 +1364,7 @@ def _test_set_mask_selection(v, a, z, selection): assert_array_equal(a, z[:]) -def test_set_mask_selection_1d(store: StorePath): +def test_set_mask_selection_1d(store: StorePath) -> None: # setup v = np.arange(1050, dtype=int) a = np.empty_like(v) @@ -1375,7 +1383,7 @@ def test_set_mask_selection_1d(store: StorePath): z.vindex[selection] = 42 -def test_set_mask_selection_2d(store: StorePath): +def test_set_mask_selection_2d(store: StorePath) -> None: # setup v = np.arange(10000, dtype=int).reshape(1000, 10) a = np.empty_like(v) @@ -1388,7 +1396,7 @@ def test_set_mask_selection_2d(store: StorePath): _test_set_mask_selection(v, a, z, ix) -def test_get_selection_out(store: StorePath): +def test_get_selection_out(store: StorePath) -> None: # basic selections a = np.arange(1050) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) @@ -1458,7 +1466,7 @@ def test_get_selection_out(store: StorePath): @pytest.mark.xfail(reason="fields are not supported in v3") -def test_get_selections_with_fields(store: StorePath): +def test_get_selections_with_fields(store: StorePath) -> None: a = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] a = np.array(a, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) z = zarr_array_from_numpy_array(store, a, chunk_shape=(2,)) @@ -1564,7 +1572,7 @@ def test_get_selections_with_fields(store: StorePath): @pytest.mark.xfail(reason="fields are not supported in v3") -def test_set_selections_with_fields(store: StorePath): +def test_set_selections_with_fields(store: StorePath) -> None: v = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] v = np.array(v, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) a = np.empty_like(v) @@ -1647,14 +1655,14 @@ def test_set_selections_with_fields(store: StorePath): assert_array_equal(a, z[:]) -def test_slice_selection_uints(): +def test_slice_selection_uints() -> None: arr = np.arange(24).reshape((4, 6)) idx = np.uint64(3) slice_sel = make_slice_selection((idx,)) assert arr[tuple(slice_sel)].shape == (1, 6) -def test_numpy_int_indexing(store: StorePath): +def test_numpy_int_indexing(store: StorePath) -> None: a = np.arange(1050) z = zarr_array_from_numpy_array(store, a, chunk_shape=(100,)) assert a[42] == z[42] @@ -1686,7 +1694,7 @@ def test_numpy_int_indexing(store: StorePath): ), ], ) -async def test_accessed_chunks(shape, chunks, ops): +async def test_accessed_chunks(shape, chunks, ops) -> None: # Test that only the required chunks are accessed during basic selection operations # shape: array shape # chunks: chunk size @@ -1759,7 +1767,7 @@ async def test_accessed_chunks(shape, chunks, ops): [[100, 200, 300], [4, 5, 6]], ], ) -def test_indexing_equals_numpy(store, selection): +def test_indexing_equals_numpy(store, selection) -> None: a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) # note: in python 3.10 a[*selection] is not valid unpacking syntax @@ -1777,7 +1785,7 @@ def test_indexing_equals_numpy(store, selection): [np.full(1000, True), [True, False] * 5], ], ) -def test_orthogonal_bool_indexing_like_numpy_ix(store, selection): +def test_orthogonal_bool_indexing_like_numpy_ix(store, selection) -> None: a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) expected = a[np.ix_(*selection)] diff --git a/tests/v3/test_store/test_memory.py b/tests/v3/test_store/test_memory.py index e64f93ad95..13aaa20bda 100644 --- a/tests/v3/test_store/test_memory.py +++ b/tests/v3/test_store/test_memory.py @@ -82,7 +82,7 @@ def test_list_prefix(self, store: GpuMemoryStore) -> None: assert True -def test_uses_dict(): +def test_uses_dict() -> None: store_dict = {} store = MemoryStore(store_dict) assert store._store_dict is store_dict diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index 7495bec8e1..e400857c45 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -48,7 +48,7 @@ def get_boto3_client() -> botocore.client.BaseClient: @pytest.fixture(autouse=True, scope="function") -def s3(s3_base) -> Generator[s3fs.S3FileSystem, None, None]: +def s3(s3_base: None) -> Generator[s3fs.S3FileSystem, None, None]: """ Quoting Martin Durant: pytest-asyncio creates a new event loop for each async test. diff --git a/tests/v3/test_store/test_stateful_store.py b/tests/v3/test_store/test_stateful_store.py index d062f9235f..1ecbd87cc1 100644 --- a/tests/v3/test_store/test_stateful_store.py +++ b/tests/v3/test_store/test_stateful_store.py @@ -9,6 +9,7 @@ precondition, rule, ) +from hypothesis.strategies import DataObject import zarr from zarr.abc.store import AccessMode, Store @@ -18,7 +19,7 @@ class SyncStoreWrapper(zarr.core.sync.SyncMixin): - def __init__(self, store: Store): + def __init__(self, store: Store) -> None: """Synchronous Store wrapper This class holds synchronous methods that map to async methods of Store classes. @@ -109,7 +110,7 @@ def __init__(self) -> None: self.prototype = default_buffer_prototype() @rule(key=paths, data=st.binary(min_size=0, max_size=100)) - def set(self, key: str, data: bytes) -> None: + def set(self, key: str, data: DataObject) -> None: note(f"(set) Setting {key!r} with {data}") assert not self.store.mode.readonly data_buf = cpu.Buffer.from_bytes(data) @@ -118,7 +119,7 @@ def set(self, key: str, data: bytes) -> None: @precondition(lambda self: len(self.model.keys()) > 0) @rule(key=paths, data=st.data()) - def get(self, key: str, data: bytes) -> None: + def get(self, key: str, data: DataObject) -> None: key = data.draw( st.sampled_from(sorted(self.model.keys())) ) # hypothesis wants to sample from sorted list @@ -128,14 +129,14 @@ def get(self, key: str, data: bytes) -> None: assert self.model[key].to_bytes() == (store_value.to_bytes()) @rule(key=paths, data=st.data()) - def get_invalid_keys(self, key: str, data: bytes) -> None: + def get_invalid_keys(self, key: str, data: DataObject) -> None: note("(get_invalid)") assume(key not in self.model.keys()) assert self.store.get(key, self.prototype) is None @precondition(lambda self: len(self.model.keys()) > 0) @rule(data=st.data()) - def get_partial_values(self, data: bytes) -> None: + def get_partial_values(self, data: DataObject) -> None: key_range = data.draw(key_ranges(keys=st.sampled_from(sorted(self.model.keys())))) note(f"(get partial) {key_range=}") obs_maybe = self.store.get_partial_values(key_range, self.prototype) @@ -162,7 +163,7 @@ def get_partial_values(self, data: bytes) -> None: @precondition(lambda self: len(self.model.keys()) > 0) @rule(data=st.data()) - def delete(self, data: bytes) -> None: + def delete(self, data: DataObject) -> None: key = data.draw(st.sampled_from(sorted(self.model.keys()))) note(f"(delete) Deleting {key=}") From f1bd70311f93991dac7aa5dd85de2f6b89aa31d6 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Fri, 13 Sep 2024 09:46:28 -0700 Subject: [PATCH 0155/1043] feature(store): V3 ZipStore (#2078) * feature(store): add basic implementation of a zip store * add zip store to array/group/sharding tests * fix sharding and skip tests that require delete * store context managers * fix test typing * add buffer_cls to store test * clean up test failures * class docstring * remove commented out check against zipstore.delete * add api integration test --------- Co-authored-by: Davis Bennett --- src/zarr/abc/store.py | 17 +- src/zarr/store/__init__.py | 11 +- src/zarr/store/local.py | 1 + src/zarr/store/memory.py | 1 + src/zarr/store/remote.py | 1 + src/zarr/store/zip.py | 223 ++++++++++++++++++++++++++ tests/v3/conftest.py | 15 +- tests/v3/test_array.py | 6 +- tests/v3/test_codecs/test_sharding.py | 18 ++- tests/v3/test_group.py | 80 +++++---- tests/v3/test_store/test_zip.py | 98 +++++++++++ 11 files changed, 413 insertions(+), 58 deletions(-) create mode 100644 src/zarr/store/zip.py create mode 100644 tests/v3/test_store/test_zip.py diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 00b81a3aad..70ac9adc17 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -11,6 +11,7 @@ class AccessMode(NamedTuple): + str: AccessModeLiteral readonly: bool overwrite: bool create: bool @@ -20,6 +21,7 @@ class AccessMode(NamedTuple): def from_literal(cls, mode: AccessModeLiteral) -> Self: if mode in ("r", "r+", "a", "w", "w-"): return cls( + str=mode, readonly=mode == "r", overwrite=mode == "w", create=mode in ("a", "w", "w-"), @@ -42,6 +44,14 @@ async def open(cls, *args: Any, **kwargs: Any) -> Self: await store._open() return store + def __enter__(self) -> Self: + """Enter a context manager that will close the store upon exiting.""" + return self + + def __exit__(self, *args: Any) -> None: + """Close the store.""" + self.close() + async def _open(self) -> None: if self._is_open: raise ValueError("store is already open") @@ -143,6 +153,12 @@ async def set(self, key: str, value: Buffer) -> None: """ ... + @property + @abstractmethod + def supports_deletes(self) -> bool: + """Does the store support deletes?""" + ... + @abstractmethod async def delete(self, key: str) -> None: """Remove a key from the store @@ -221,7 +237,6 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: def close(self) -> None: """Close the store.""" self._is_open = False - pass @runtime_checkable diff --git a/src/zarr/store/__init__.py b/src/zarr/store/__init__.py index 3a9e8201e8..47bbccd66e 100644 --- a/src/zarr/store/__init__.py +++ b/src/zarr/store/__init__.py @@ -2,5 +2,14 @@ from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore from zarr.store.remote import RemoteStore +from zarr.store.zip import ZipStore -__all__ = ["StorePath", "StoreLike", "make_store_path", "RemoteStore", "LocalStore", "MemoryStore"] +__all__ = [ + "StorePath", + "StoreLike", + "make_store_path", + "RemoteStore", + "LocalStore", + "MemoryStore", + "ZipStore", +] diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index b69b86dc4c..5fd48c2db0 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -73,6 +73,7 @@ def _put( class LocalStore(Store): supports_writes: bool = True + supports_deletes: bool = True supports_partial_writes: bool = True supports_listing: bool = True diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 4f2627bf67..89e7ced31e 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -18,6 +18,7 @@ # When that is done, the `MemoryStore` will just be a store that wraps a dict. class MemoryStore(Store): supports_writes: bool = True + supports_deletes: bool = True supports_partial_writes: bool = True supports_listing: bool = True diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index f883832bc7..3907ac3cc2 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -20,6 +20,7 @@ class RemoteStore(Store): # based on FSSpec supports_writes: bool = True + supports_deletes: bool = True supports_partial_writes: bool = False supports_listing: bool = True diff --git a/src/zarr/store/zip.py b/src/zarr/store/zip.py new file mode 100644 index 0000000000..15473aa674 --- /dev/null +++ b/src/zarr/store/zip.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import os +import threading +import time +import zipfile +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +from zarr.abc.store import Store +from zarr.core.buffer import Buffer, BufferPrototype + +if TYPE_CHECKING: + from collections.abc import AsyncGenerator + +ZipStoreAccessModeLiteral = Literal["r", "w", "a"] + + +class ZipStore(Store): + """ + Storage class using a ZIP file. + + Parameters + ---------- + path : string + Location of file. + compression : integer, optional + Compression method to use when writing to the archive. + allowZip64 : bool, optional + If True (the default) will create ZIP files that use the ZIP64 + extensions when the zipfile is larger than 2 GiB. If False + will raise an exception when the ZIP file would require ZIP64 + extensions. + mode : string, optional + One of 'r' to read an existing file, 'w' to truncate and write a new + file, 'a' to append to an existing file, or 'x' to exclusively create + and write a new file. + """ + + supports_writes: bool = True + supports_deletes: bool = False + supports_partial_writes: bool = False + supports_listing: bool = True + + path: Path + compression: int + allowZip64: bool + + _zf: zipfile.ZipFile + _lock: threading.RLock + + def __init__( + self, + path: Path | str, + *, + mode: ZipStoreAccessModeLiteral = "r", + compression: int = zipfile.ZIP_STORED, + allowZip64: bool = True, + ): + super().__init__(mode=mode) + + if isinstance(path, str): + path = Path(path) + assert isinstance(path, Path) + self.path = path # root? + + self._zmode = mode + self.compression = compression + self.allowZip64 = allowZip64 + + async def _open(self) -> None: + if self._is_open: + raise ValueError("store is already open") + + self._lock = threading.RLock() + + self._zf = zipfile.ZipFile( + self.path, + mode=self._zmode, + compression=self.compression, + allowZip64=self.allowZip64, + ) + + self._is_open = True + + def close(self) -> None: + super().close() + with self._lock: + self._zf.close() + + async def clear(self) -> None: + with self._lock: + self._check_writable() + self._zf.close() + os.remove(self.path) + self._zf = zipfile.ZipFile( + self.path, mode="w", compression=self.compression, allowZip64=self.allowZip64 + ) + + async def empty(self) -> bool: + with self._lock: + if self._zf.namelist(): + return False + else: + return True + + def __str__(self) -> str: + return f"zip://{self.path}" + + def __repr__(self) -> str: + return f"ZipStore({str(self)!r})" + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) and self.path == other.path + + def _get( + self, + key: str, + prototype: BufferPrototype, + byte_range: tuple[int | None, int | None] | None = None, + ) -> Buffer | None: + try: + with self._zf.open(key) as f: # will raise KeyError + if byte_range is None: + return prototype.buffer.from_bytes(f.read()) + start, length = byte_range + if start: + if start < 0: + start = f.seek(start, os.SEEK_END) + start + else: + start = f.seek(start, os.SEEK_SET) + if length: + return prototype.buffer.from_bytes(f.read(length)) + else: + return prototype.buffer.from_bytes(f.read()) + except KeyError: + return None + + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: tuple[int | None, int | None] | None = None, + ) -> Buffer | None: + assert isinstance(key, str) + + with self._lock: + return self._get(key, prototype=prototype, byte_range=byte_range) + + async def get_partial_values( + self, + prototype: BufferPrototype, + key_ranges: list[tuple[str, tuple[int | None, int | None]]], + ) -> list[Buffer | None]: + out = [] + with self._lock: + for key, byte_range in key_ranges: + out.append(self._get(key, prototype=prototype, byte_range=byte_range)) + return out + + def _set(self, key: str, value: Buffer) -> None: + # generally, this should be called inside a lock + keyinfo = zipfile.ZipInfo(filename=key, date_time=time.localtime(time.time())[:6]) + keyinfo.compress_type = self.compression + if keyinfo.filename[-1] == os.sep: + keyinfo.external_attr = 0o40775 << 16 # drwxrwxr-x + keyinfo.external_attr |= 0x10 # MS-DOS directory flag + else: + keyinfo.external_attr = 0o644 << 16 # ?rw-r--r-- + self._zf.writestr(keyinfo, value.to_bytes()) + + async def set(self, key: str, value: Buffer) -> None: + self._check_writable() + assert isinstance(key, str) + if not isinstance(value, Buffer): + raise TypeError("ZipStore.set(): `value` must a Buffer instance") + with self._lock: + self._set(key, value) + + async def set_partial_values(self, key_start_values: list[tuple[str, int, bytes]]) -> None: + raise NotImplementedError + + async def delete(self, key: str) -> None: + raise NotImplementedError + + async def exists(self, key: str) -> bool: + with self._lock: + try: + self._zf.getinfo(key) + except KeyError: + return False + else: + return True + + async def list(self) -> AsyncGenerator[str, None]: + with self._lock: + for key in self._zf.namelist(): + yield key + + async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + async for key in self.list(): + if key.startswith(prefix): + yield key + + async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + if prefix.endswith("/"): + prefix = prefix[:-1] + + keys = self._zf.namelist() + seen = set() + if prefix == "": + keys_unique = set(k.split("/")[0] for k in keys) + for key in keys_unique: + if key not in seen: + seen.add(key) + yield key + else: + for key in keys: + if key.startswith(prefix + "/") and key != prefix: + k = key.removeprefix(prefix + "/").split("/")[0] + if k not in seen: + seen.add(k) + yield k diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index b1308f058f..41cd359346 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -10,7 +10,7 @@ from hypothesis import HealthCheck, Verbosity, settings from zarr import AsyncGroup, config -from zarr.store import LocalStore, MemoryStore, StorePath +from zarr.store import LocalStore, MemoryStore, StorePath, ZipStore from zarr.store.remote import RemoteStore if TYPE_CHECKING: @@ -25,14 +25,16 @@ async def parse_store( - store: Literal["local", "memory", "remote"], path: str -) -> LocalStore | MemoryStore | RemoteStore: + store: Literal["local", "memory", "remote", "zip"], path: str +) -> LocalStore | MemoryStore | RemoteStore | ZipStore: if store == "local": return await LocalStore.open(path, mode="w") if store == "memory": return await MemoryStore.open(mode="w") if store == "remote": return await RemoteStore.open(url=path, mode="w") + if store == "zip": + return await ZipStore.open(path + "/zarr.zip", mode="w") raise AssertionError @@ -64,6 +66,11 @@ async def memory_store() -> MemoryStore: return await MemoryStore.open(mode="w") +@pytest.fixture(scope="function") +async def zip_store(tmpdir: LEGACY_PATH) -> ZipStore: + return await ZipStore.open(str(tmpdir / "zarr.zip"), mode="w") + + @pytest.fixture(scope="function") async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: param = request.param @@ -73,7 +80,7 @@ async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: @dataclass class AsyncGroupRequest: zarr_format: ZarrFormat - store: Literal["local", "remote", "memory"] + store: Literal["local", "remote", "memory", "zip"] attributes: dict[str, Any] = field(default_factory=dict) diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index fb726757dc..cd20ab6e58 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -10,7 +10,7 @@ from zarr.store.common import StorePath -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) @pytest.mark.parametrize("exists_ok", [True, False]) @pytest.mark.parametrize("extant_node", ["array", "group"]) @@ -59,7 +59,7 @@ def test_array_creation_existing_node( ) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) def test_array_name_properties_no_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat @@ -70,7 +70,7 @@ def test_array_name_properties_no_group( assert arr.basename is None -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) def test_array_name_properties_with_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat diff --git a/tests/v3/test_codecs/test_sharding.py b/tests/v3/test_codecs/test_sharding.py index 8a602c2c8b..bd8aab5e03 100644 --- a/tests/v3/test_codecs/test_sharding.py +++ b/tests/v3/test_codecs/test_sharding.py @@ -21,7 +21,7 @@ from .test_codecs import _AsyncArrayProxy, order_from_dim -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) @pytest.mark.parametrize("index_location", ["start", "end"]) @pytest.mark.parametrize( "array_fixture", @@ -76,7 +76,7 @@ def test_sharding( @pytest.mark.parametrize("index_location", ["start", "end"]) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) @pytest.mark.parametrize( "array_fixture", [ @@ -126,7 +126,7 @@ def test_sharding_partial( indirect=["array_fixture"], ) @pytest.mark.parametrize("index_location", ["start", "end"]) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) def test_sharding_partial_read( store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation ) -> None: @@ -163,7 +163,7 @@ def test_sharding_partial_read( indirect=["array_fixture"], ) @pytest.mark.parametrize("index_location", ["start", "end"]) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) def test_sharding_partial_overwrite( store: Store, array_fixture: npt.NDArray[Any], index_location: ShardingCodecIndexLocation ) -> None: @@ -214,7 +214,7 @@ def test_sharding_partial_overwrite( "inner_index_location", ["start", "end"], ) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) def test_nested_sharding( store: Store, array_fixture: npt.NDArray[Any], @@ -247,7 +247,7 @@ def test_nested_sharding( assert np.array_equal(data, read_data) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) def test_open_sharding(store: Store) -> None: path = "open_sharding" spath = StorePath(store, path) @@ -272,7 +272,7 @@ def test_open_sharding(store: Store) -> None: assert a.metadata == b.metadata -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) def test_write_partial_sharded_chunks(store: Store) -> None: data = np.arange(0, 16 * 16, dtype="uint16").reshape((16, 16)) spath = StorePath(store) @@ -296,8 +296,10 @@ def test_write_partial_sharded_chunks(store: Store) -> None: assert np.array_equal(a[0:16, 0:16], data) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) async def test_delete_empty_shards(store: Store) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") path = "delete_empty_shards" spath = StorePath(store, path) a = await AsyncArray.create( diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index f688fc81d4..a62f367351 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -7,6 +7,7 @@ import zarr.api.asynchronous from zarr import Array, AsyncArray, AsyncGroup, Group +from zarr.abc.store import Store from zarr.api.synchronous import open_group from zarr.core.buffer import default_buffer_prototype from zarr.core.common import JSON, ZarrFormat @@ -22,10 +23,10 @@ from _pytest.compat import LEGACY_PATH -@pytest.fixture(params=["local", "memory"]) -async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> LocalStore | MemoryStore: +@pytest.fixture(params=["local", "memory", "zip"]) +async def store(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> Store: result = await parse_store(request.param, str(tmpdir)) - if not isinstance(result, MemoryStore | LocalStore): + if not isinstance(result, Store): raise TypeError("Wrong store class returned by test fixture! got " + result + " instead") return result @@ -46,7 +47,7 @@ def zarr_format(request: pytest.FixtureRequest) -> ZarrFormat: return cast(ZarrFormat, result) -def test_group_init(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: +def test_group_init(store: Store, zarr_format: ZarrFormat) -> None: """ Test that initializing a group from an asyncgroup works. """ @@ -55,7 +56,7 @@ def test_group_init(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> assert group._async_group == agroup -def test_group_name_properties(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: +def test_group_name_properties(store: Store, zarr_format: ZarrFormat) -> None: """ Test basic properties of groups """ @@ -75,7 +76,7 @@ def test_group_name_properties(store: LocalStore | MemoryStore, zarr_format: Zar assert bar.basename == "bar" -def test_group_members(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_members(store: Store, zarr_format: ZarrFormat) -> None: """ Test that `Group.members` returns correct values, i.e. the arrays and groups (explicit and implicit) contained in that group. @@ -133,7 +134,7 @@ def test_group_members(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) members_observed = group.members(max_depth=-1) -def test_group(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group(store: Store, zarr_format: ZarrFormat) -> None: """ Test basic Group routines. """ @@ -175,9 +176,7 @@ def test_group(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None assert dict(bar3.attrs) == {"baz": "qux", "name": "bar"} -def test_group_create( - store: MemoryStore | LocalStore, exists_ok: bool, zarr_format: ZarrFormat -) -> None: +def test_group_create(store: Store, exists_ok: bool, zarr_format: ZarrFormat) -> None: """ Test that `Group.create` works as expected. """ @@ -193,9 +192,7 @@ def test_group_create( ) -def test_group_open( - store: MemoryStore | LocalStore, zarr_format: ZarrFormat, exists_ok: bool -) -> None: +def test_group_open(store: Store, zarr_format: ZarrFormat, exists_ok: bool) -> None: """ Test the `Group.open` method. """ @@ -227,7 +224,7 @@ def test_group_open( assert group_created_again.store_path == spath -def test_group_getitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_getitem(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__getitem__` method. """ @@ -242,10 +239,12 @@ def test_group_getitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) group["nope"] -def test_group_delitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_delitem(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__delitem__` method. """ + if not store.supports_deletes: + pytest.skip("store does not support deletes") group = Group.create(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") @@ -263,7 +262,7 @@ def test_group_delitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) group["subarray"] -def test_group_iter(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_iter(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__iter__` method. """ @@ -273,7 +272,7 @@ def test_group_iter(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> [x for x in group] # type: ignore -def test_group_len(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_len(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__len__` method. """ @@ -283,7 +282,7 @@ def test_group_len(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> len(group) # type: ignore -def test_group_setitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_setitem(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__setitem__` method. """ @@ -292,7 +291,7 @@ def test_group_setitem(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) group["key"] = 10 -def test_group_contains(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_contains(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__contains__` method """ @@ -302,7 +301,7 @@ def test_group_contains(store: MemoryStore | LocalStore, zarr_format: ZarrFormat assert "foo" in group -def test_group_subgroups(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_subgroups(store: Store, zarr_format: ZarrFormat) -> None: """ Test the behavior of `Group` methods for accessing subgroups, namely `Group.group_keys` and `Group.groups` """ @@ -317,7 +316,7 @@ def test_group_subgroups(store: MemoryStore | LocalStore, zarr_format: ZarrForma assert all(a in subgroups_observed for a in subgroups_expected) -def test_group_subarrays(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_subarrays(store: Store, zarr_format: ZarrFormat) -> None: """ Test the behavior of `Group` methods for accessing subgroups, namely `Group.group_keys` and `Group.groups` """ @@ -332,7 +331,7 @@ def test_group_subarrays(store: MemoryStore | LocalStore, zarr_format: ZarrForma assert all(a in subarrays_observed for a in subarrays_expected) -def test_group_update_attributes(store: MemoryStore | LocalStore, zarr_format: ZarrFormat) -> None: +def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: """ Test the behavior of `Group.update_attributes` """ @@ -344,9 +343,7 @@ def test_group_update_attributes(store: MemoryStore | LocalStore, zarr_format: Z assert new_group.attrs == new_attrs -async def test_group_update_attributes_async( - store: MemoryStore | LocalStore, zarr_format: ZarrFormat -) -> None: +async def test_group_update_attributes_async(store: Store, zarr_format: ZarrFormat) -> None: """ Test the behavior of `Group.update_attributes_async` """ @@ -360,7 +357,7 @@ async def test_group_update_attributes_async( @pytest.mark.parametrize("method", ["create_array", "array"]) def test_group_create_array( - store: MemoryStore | LocalStore, + store: Store, zarr_format: ZarrFormat, exists_ok: bool, method: Literal["create_array", "array"], @@ -393,12 +390,12 @@ def test_group_create_array( assert np.array_equal(array[:], data) -@pytest.mark.parametrize("store", ("local", "memory"), indirect=["store"]) +@pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) @pytest.mark.parametrize("exists_ok", [True, False]) @pytest.mark.parametrize("extant_node", ["array", "group"]) def test_group_creation_existing_node( - store: LocalStore | MemoryStore, + store: Store, zarr_format: ZarrFormat, exists_ok: bool, extant_node: Literal["array", "group"], @@ -441,7 +438,7 @@ def test_group_creation_existing_node( async def test_asyncgroup_create( - store: MemoryStore | LocalStore, + store: Store, exists_ok: bool, zarr_format: ZarrFormat, ) -> None: @@ -482,14 +479,14 @@ async def test_asyncgroup_create( ) -async def test_asyncgroup_attrs(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: +async def test_asyncgroup_attrs(store: Store, zarr_format: ZarrFormat) -> None: attributes = {"foo": 100} agroup = await AsyncGroup.create(store, zarr_format=zarr_format, attributes=attributes) assert agroup.attrs == agroup.metadata.attributes == attributes -async def test_asyncgroup_info(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: +async def test_asyncgroup_info(store: Store, zarr_format: ZarrFormat) -> None: agroup = await AsyncGroup.create( # noqa store, zarr_format=zarr_format, @@ -499,7 +496,7 @@ async def test_asyncgroup_info(store: LocalStore | MemoryStore, zarr_format: Zar async def test_asyncgroup_open( - store: LocalStore | MemoryStore, + store: Store, zarr_format: ZarrFormat, ) -> None: """ @@ -520,7 +517,7 @@ async def test_asyncgroup_open( async def test_asyncgroup_open_wrong_format( - store: LocalStore | MemoryStore, + store: Store, zarr_format: ZarrFormat, ) -> None: _ = await AsyncGroup.create(store=store, exists_ok=False, zarr_format=zarr_format) @@ -546,7 +543,7 @@ async def test_asyncgroup_open_wrong_format( {"zarr_format": 2, "attributes": {"foo": 100}}, ), ) -def test_asyncgroup_from_dict(store: MemoryStore | LocalStore, data: dict[str, Any]) -> None: +def test_asyncgroup_from_dict(store: Store, data: dict[str, Any]) -> None: """ Test that we can create an AsyncGroup from a dict """ @@ -561,7 +558,7 @@ def test_asyncgroup_from_dict(store: MemoryStore | LocalStore, data: dict[str, A # todo: replace this with a declarative API where we model a full hierarchy -async def test_asyncgroup_getitem(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: +async def test_asyncgroup_getitem(store: Store, zarr_format: ZarrFormat) -> None: """ Create an `AsyncGroup`, then create members of that group, and ensure that we can access those members via the `AsyncGroup.getitem` method. @@ -583,7 +580,10 @@ async def test_asyncgroup_getitem(store: LocalStore | MemoryStore, zarr_format: await agroup.getitem("foo") -async def test_asyncgroup_delitem(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: +async def test_asyncgroup_delitem(store: Store, zarr_format: ZarrFormat) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") + agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) array_name = "sub_array" _ = await agroup.create_array( @@ -613,7 +613,7 @@ async def test_asyncgroup_delitem(store: LocalStore | MemoryStore, zarr_format: async def test_asyncgroup_create_group( - store: LocalStore | MemoryStore, + store: Store, zarr_format: ZarrFormat, ) -> None: agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) @@ -629,7 +629,7 @@ async def test_asyncgroup_create_group( async def test_asyncgroup_create_array( - store: LocalStore | MemoryStore, zarr_format: ZarrFormat, exists_ok: bool + store: Store, zarr_format: ZarrFormat, exists_ok: bool ) -> None: """ Test that the AsyncGroup.create_array method works correctly. We ensure that array properties @@ -667,9 +667,7 @@ async def test_asyncgroup_create_array( assert subnode.metadata.zarr_format == zarr_format -async def test_asyncgroup_update_attributes( - store: LocalStore | MemoryStore, zarr_format: ZarrFormat -) -> None: +async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: """ Test that the AsyncGroup.update_attributes method works correctly. """ diff --git a/tests/v3/test_store/test_zip.py b/tests/v3/test_store/test_zip.py new file mode 100644 index 0000000000..7c332e9a2e --- /dev/null +++ b/tests/v3/test_store/test_zip.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import os +import tempfile +from typing import TYPE_CHECKING + +import numpy as np +import pytest + +import zarr +from zarr.abc.store import AccessMode +from zarr.core.buffer import Buffer, cpu, default_buffer_prototype +from zarr.store.zip import ZipStore +from zarr.testing.store import StoreTests + +if TYPE_CHECKING: + from collections.abc import Coroutine + from typing import Any + + +class TestZipStore(StoreTests[ZipStore, cpu.Buffer]): + store_cls = ZipStore + buffer_cls = cpu.Buffer + + @pytest.fixture(scope="function") + def store_kwargs(self, request) -> dict[str, str | bool]: + fd, temp_path = tempfile.mkstemp() + os.close(fd) + + return {"path": temp_path, "mode": "w"} + + def get(self, store: ZipStore, key: str) -> Buffer: + return store._get(key, prototype=default_buffer_prototype()) + + def set(self, store: ZipStore, key: str, value: Buffer) -> None: + return store._set(key, value) + + def test_store_mode(self, store: ZipStore, store_kwargs: dict[str, Any]) -> None: + assert store.mode == AccessMode.from_literal(store_kwargs["mode"]) + assert not store.mode.readonly + + async def test_not_writable_store_raises(self, store_kwargs: dict[str, Any]) -> None: + # we need to create the zipfile in write mode before switching to read mode + store = await self.store_cls.open(**store_kwargs) + store.close() + + kwargs = {**store_kwargs, "mode": "r"} + store = await self.store_cls.open(**kwargs) + assert store.mode == AccessMode.from_literal("r") + assert store.mode.readonly + + # set + with pytest.raises(ValueError): + await store.set("foo", cpu.Buffer.from_bytes(b"bar")) + + def test_store_repr(self, store: ZipStore) -> None: + assert str(store) == f"zip://{store.path!s}" + + def test_store_supports_writes(self, store: ZipStore) -> None: + assert store.supports_writes + + def test_store_supports_partial_writes(self, store: ZipStore) -> None: + assert store.supports_partial_writes is False + + def test_store_supports_listing(self, store: ZipStore) -> None: + assert store.supports_listing + + def test_delete(self, store: ZipStore) -> Coroutine[Any, Any, None]: + pass + + def test_api_integration(self, store: ZipStore) -> None: + root = zarr.open_group(store=store) + + data = np.arange(10000, dtype=np.uint16).reshape(100, 100) + z = root.create_array( + shape=data.shape, chunks=(10, 10), name="foo", dtype=np.uint16, fill_value=99 + ) + z[:] = data + + assert np.array_equal(data, z[:]) + + # you can overwrite existing chunks but zipfile will issue a warning + with pytest.warns(UserWarning, match="Duplicate name: 'foo/c/0/0'"): + z[0, 0] = 100 + + # TODO: assigning an entire chunk to fill value ends up deleting the chunk which is not supported + # a work around will be needed here. + with pytest.raises(NotImplementedError): + z[0:10, 0:10] = 99 + + bar = root.create_group("bar", attributes={"hello": "world"}) + assert "hello" in dict(bar.attrs) + + # keys cannot be deleted + with pytest.raises(NotImplementedError): + del root["bar"] + + store.close() From ceb3b361fb912dc748ac871018bcf7c757e4719d Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 14 Sep 2024 09:09:47 -0700 Subject: [PATCH 0156/1043] test: check that store, array, and group classes are serializable (#2006) * test: check that store, array, and group classes are serializable w/ pickle and can be dependably roundtripped * raise if MemoryStore is pickled * Apply suggestions from code review Co-authored-by: Davis Bennett * fix typos * new buffer __eq__ * pickle support for zip store --------- Co-authored-by: Davis Bennett --- src/zarr/abc/store.py | 5 + src/zarr/core/buffer/core.py | 6 ++ src/zarr/store/memory.py | 16 ++- src/zarr/store/remote.py | 10 ++ src/zarr/store/zip.py | 15 ++- src/zarr/testing/store.py | 14 +++ tests/v3/test_array.py | 36 ++++++- tests/v3/test_group.py | 165 +++-------------------------- tests/v3/test_store/test_memory.py | 52 ++------- tests/v3/test_store/test_remote.py | 2 +- 10 files changed, 127 insertions(+), 194 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 70ac9adc17..95d55a2ce0 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -83,6 +83,11 @@ def _check_writable(self) -> None: if self.mode.readonly: raise ValueError("store mode does not support writing") + @abstractmethod + def __eq__(self, value: object) -> bool: + """Equality comparison.""" + ... + @abstractmethod async def get( self, diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index ff26478ca9..0c6d966db9 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -281,6 +281,12 @@ def __add__(self, other: Buffer) -> Self: """Concatenate two buffers""" ... + def __eq__(self, other: object) -> bool: + # Another Buffer class can override this to choose a more efficient path + return isinstance(other, Buffer) and np.array_equal( + self.as_numpy_array(), other.as_numpy_array() + ) + class NDBuffer: """An n-dimensional memory block diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 89e7ced31e..13e289f374 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from collections.abc import AsyncGenerator, MutableMapping +from typing import TYPE_CHECKING, Any from zarr.abc.store import Store from zarr.core.buffer import Buffer, gpu @@ -47,6 +48,19 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"MemoryStore({str(self)!r})" + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self._store_dict == other._store_dict + and self.mode == other.mode + ) + + def __setstate__(self, state: Any) -> None: + raise NotImplementedError(f"{type(self)} cannot be pickled") + + def __getstate__(self) -> None: + raise NotImplementedError(f"{type(self)} cannot be pickled") + async def get( self, key: str, diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index 3907ac3cc2..e3e2ba3447 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -51,6 +51,7 @@ def __init__( this must not be used. """ super().__init__(mode=mode) + self._storage_options = storage_options if isinstance(url, str): self._url = url.rstrip("/") self._fs, _path = fsspec.url_to_fs(url, **storage_options) @@ -91,6 +92,15 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"" + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, type(self)) + and self.path == other.path + and self.mode == other.mode + and self._url == other._url + # and self._storage_options == other._storage_options # FIXME: this isn't working for some reason + ) + async def get( self, key: str, diff --git a/src/zarr/store/zip.py b/src/zarr/store/zip.py index 15473aa674..ea31ad934a 100644 --- a/src/zarr/store/zip.py +++ b/src/zarr/store/zip.py @@ -5,7 +5,7 @@ import time import zipfile from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from zarr.abc.store import Store from zarr.core.buffer import Buffer, BufferPrototype @@ -68,7 +68,7 @@ def __init__( self.compression = compression self.allowZip64 = allowZip64 - async def _open(self) -> None: + def _sync_open(self) -> None: if self._is_open: raise ValueError("store is already open") @@ -83,6 +83,17 @@ async def _open(self) -> None: self._is_open = True + async def _open(self) -> None: + self._sync_open() + + def __getstate__(self) -> tuple[Path, ZipStoreAccessModeLiteral, int, bool]: + return self.path, self._zmode, self.compression, self.allowZip64 + + def __setstate__(self, state: Any) -> None: + self.path, self._zmode, self.compression, self.allowZip64 = state + self._is_open = False + self._sync_open() + def close(self) -> None: super().close() with self._lock: diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 65d7751a0d..a08b6960db 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -1,3 +1,4 @@ +import pickle from typing import Any, Generic, TypeVar import pytest @@ -48,6 +49,19 @@ def test_store_type(self, store: S) -> None: assert isinstance(store, Store) assert isinstance(store, self.store_cls) + def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None: + # check self equality + assert store == store + + # check store equality with same inputs + # asserting this is important for being able to compare (de)serialized stores + store2 = self.store_cls(**store_kwargs) + assert store == store2 + + def test_serizalizable_store(self, store: S) -> None: + foo = pickle.dumps(store) + assert pickle.loads(foo) == store + def test_store_mode(self, store: S, store_kwargs: dict[str, Any]) -> None: assert store.mode == AccessMode.from_literal("r+") assert not store.mode.readonly diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index cd20ab6e58..11be51682c 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -1,9 +1,10 @@ +import pickle from typing import Literal import numpy as np import pytest -from zarr import Array, Group +from zarr import Array, AsyncArray, Group from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.store import LocalStore, MemoryStore @@ -135,3 +136,36 @@ def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str assert arr.fill_value == np.dtype(dtype_str).type(fill_value) assert arr.fill_value.dtype == arr.dtype + + +@pytest.mark.parametrize("store", ("local",), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_serializable_async_array( + store: LocalStore | MemoryStore, zarr_format: ZarrFormat +) -> None: + expected = await AsyncArray.create( + store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" + ) + # await expected.setitems(list(range(100))) + + p = pickle.dumps(expected) + actual = pickle.loads(p) + + assert actual == expected + # np.testing.assert_array_equal(await actual.getitem(slice(None)), await expected.getitem(slice(None))) + # TODO: uncomment the parts of this test that will be impacted by the config/prototype changes in flight + + +@pytest.mark.parametrize("store", ("local",), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +def test_serializable_sync_array(store: LocalStore, zarr_format: ZarrFormat) -> None: + expected = Array.create( + store=store, shape=(100,), chunks=(10,), zarr_format=zarr_format, dtype="i4" + ) + expected[:] = list(range(100)) + + p = pickle.dumps(expected) + actual = pickle.loads(p) + + assert actual == expected + np.testing.assert_array_equal(actual[:], expected[:]) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index a62f367351..94b839a186 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -1,20 +1,19 @@ from __future__ import annotations +import pickle from typing import TYPE_CHECKING, Any, Literal, cast import numpy as np import pytest -import zarr.api.asynchronous from zarr import Array, AsyncArray, AsyncGroup, Group from zarr.abc.store import Store -from zarr.api.synchronous import open_group from zarr.core.buffer import default_buffer_prototype from zarr.core.common import JSON, ZarrFormat from zarr.core.group import GroupMetadata from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError -from zarr.store import LocalStore, MemoryStore, StorePath +from zarr.store import LocalStore, StorePath from zarr.store.common import make_store_path from .conftest import parse_store @@ -681,152 +680,22 @@ async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrForma assert agroup_new_attributes.attrs == attributes_new -async def test_group_members_async(store: LocalStore | MemoryStore) -> None: - group = AsyncGroup( - GroupMetadata(), - store_path=StorePath(store=store, path="root"), - ) - a0 = await group.create_array("a0", shape=(1,)) - g0 = await group.create_group("g0") - a1 = await g0.create_array("a1", shape=(1,)) - g1 = await g0.create_group("g1") - a2 = await g1.create_array("a2", shape=(1,)) - g2 = await g1.create_group("g2") - - # immediate children - children = sorted([x async for x in group.members()], key=lambda x: x[0]) - assert children == [ - ("a0", a0), - ("g0", g0), - ] - - nmembers = await group.nmembers() - assert nmembers == 2 - - # partial - children = sorted([x async for x in group.members(max_depth=1)], key=lambda x: x[0]) - expected = [ - ("a0", a0), - ("g0", g0), - ("g0/a1", a1), - ("g0/g1", g1), - ] - assert children == expected - nmembers = await group.nmembers(max_depth=1) - assert nmembers == 4 - - # all children - all_children = sorted([x async for x in group.members(max_depth=None)], key=lambda x: x[0]) - expected = [ - ("a0", a0), - ("g0", g0), - ("g0/a1", a1), - ("g0/g1", g1), - ("g0/g1/a2", a2), - ("g0/g1/g2", g2), - ] - assert all_children == expected - - nmembers = await group.nmembers(max_depth=None) - assert nmembers == 6 - - with pytest.raises(ValueError, match="max_depth"): - [x async for x in group.members(max_depth=-1)] - - -async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) - - # create foo group - _ = await root.create_group("foo", attributes={"foo": 100}) - - # test that we can get the group using require_group - foo_group = await root.require_group("foo") - assert foo_group.attrs == {"foo": 100} - - # test that we can get the group using require_group and overwrite=True - foo_group = await root.require_group("foo", overwrite=True) - - _ = await foo_group.create_array( - "bar", shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} +@pytest.mark.parametrize("store", ("local",), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +async def test_serializable_async_group(store: LocalStore, zarr_format: ZarrFormat) -> None: + expected = await AsyncGroup.create( + store=store, attributes={"foo": 999}, zarr_format=zarr_format ) + p = pickle.dumps(expected) + actual = pickle.loads(p) + assert actual == expected - # test that overwriting a group w/ children fails - # TODO: figure out why ensure_no_existing_node is not catching the foo.bar array - # - # with pytest.raises(ContainsArrayError): - # await root.require_group("foo", overwrite=True) - - # test that requiring a group where an array is fails - with pytest.raises(TypeError): - await foo_group.require_group("bar") - - -async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) - # create foo group - _ = await root.create_group("foo", attributes={"foo": 100}) - # create bar group - _ = await root.create_group("bar", attributes={"bar": 200}) - - foo_group, bar_group = await root.require_groups("foo", "bar") - assert foo_group.attrs == {"foo": 100} - assert bar_group.attrs == {"bar": 200} - - # get a mix of existing and new groups - foo_group, spam_group = await root.require_groups("foo", "spam") - assert foo_group.attrs == {"foo": 100} - assert spam_group.attrs == {} - - # no names - no_group = await root.require_groups() - assert no_group == () - - -async def test_create_dataset(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) - with pytest.warns(DeprecationWarning): - foo = await root.create_dataset("foo", shape=(10,), dtype="uint8") - assert foo.shape == (10,) - - with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): - await root.create_dataset("foo", shape=(100,), dtype="int8") - - _ = await root.create_group("bar") - with pytest.raises(ContainsGroupError), pytest.warns(DeprecationWarning): - await root.create_dataset("bar", shape=(100,), dtype="int8") - - -async def test_require_array(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) - foo1 = await root.require_array("foo", shape=(10,), dtype="i8", attributes={"foo": 101}) - assert foo1.attrs == {"foo": 101} - foo2 = await root.require_array("foo", shape=(10,), dtype="i8") - assert foo2.attrs == {"foo": 101} - - # exact = False - _ = await root.require_array("foo", shape=10, dtype="f8") - - # errors w/ exact True - with pytest.raises(TypeError, match="Incompatible dtype"): - await root.require_array("foo", shape=(10,), dtype="f8", exact=True) - - with pytest.raises(TypeError, match="Incompatible shape"): - await root.require_array("foo", shape=(100, 100), dtype="i8") - - with pytest.raises(TypeError, match="Incompatible dtype"): - await root.require_array("foo", shape=(10,), dtype="f4") - - _ = await root.create_group("bar") - with pytest.raises(TypeError, match="Incompatible object"): - await root.require_array("bar", shape=(10,), dtype="int8") - - -async def test_open_mutable_mapping(): - group = await zarr.api.asynchronous.open_group(store={}, mode="w") - assert isinstance(group.store_path.store, MemoryStore) +@pytest.mark.parametrize("store", ("local",), indirect=["store"]) +@pytest.mark.parametrize("zarr_format", (2, 3)) +def test_serializable_sync_group(store: LocalStore, zarr_format: ZarrFormat) -> None: + expected = Group.create(store=store, attributes={"foo": 999}, zarr_format=zarr_format) + p = pickle.dumps(expected) + actual = pickle.loads(p) -def test_open_mutable_mapping_sync(): - group = open_group(store={}, mode="w") - assert isinstance(group.store_path.store, MemoryStore) + assert actual == expected diff --git a/tests/v3/test_store/test_memory.py b/tests/v3/test_store/test_memory.py index 13aaa20bda..04d17eb240 100644 --- a/tests/v3/test_store/test_memory.py +++ b/tests/v3/test_store/test_memory.py @@ -1,11 +1,12 @@ from __future__ import annotations +import pickle + import pytest -from zarr.core.buffer import Buffer, cpu, gpu -from zarr.store.memory import GpuMemoryStore, MemoryStore +from zarr.core.buffer import Buffer, cpu +from zarr.store.memory import MemoryStore from zarr.testing.store import StoreTests -from zarr.testing.utils import gpu_test class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]): @@ -46,43 +47,12 @@ def test_store_supports_partial_writes(self, store: MemoryStore) -> None: def test_list_prefix(self, store: MemoryStore) -> None: assert True + def test_serizalizable_store(self, store: MemoryStore) -> None: + with pytest.raises(NotImplementedError): + store.__getstate__() -@gpu_test -class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): - store_cls = GpuMemoryStore - buffer_cls = gpu.Buffer - - def set(self, store: GpuMemoryStore, key: str, value: Buffer) -> None: - store._store_dict[key] = value - - def get(self, store: MemoryStore, key: str) -> Buffer: - return store._store_dict[key] - - @pytest.fixture(scope="function", params=[None, {}]) - def store_kwargs(self, request) -> dict[str, str | None | dict[str, Buffer]]: - return {"store_dict": request.param, "mode": "r+"} - - @pytest.fixture(scope="function") - def store(self, store_kwargs: str | None | dict[str, gpu.Buffer]) -> GpuMemoryStore: - return self.store_cls(**store_kwargs) - - def test_store_repr(self, store: GpuMemoryStore) -> None: - assert str(store) == f"gpumemory://{id(store._store_dict)}" - - def test_store_supports_writes(self, store: GpuMemoryStore) -> None: - assert store.supports_writes - - def test_store_supports_listing(self, store: GpuMemoryStore) -> None: - assert store.supports_listing - - def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: - assert store.supports_partial_writes - - def test_list_prefix(self, store: GpuMemoryStore) -> None: - assert True - + with pytest.raises(NotImplementedError): + store.__setstate__({}) -def test_uses_dict() -> None: - store_dict = {} - store = MemoryStore(store_dict) - assert store._store_dict is store_dict + with pytest.raises(NotImplementedError): + pickle.dumps(store) diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index e400857c45..afa991209f 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -109,7 +109,7 @@ def store_kwargs(self, request) -> dict[str, str | bool]: anon = False mode = "r+" if request.param == "use_upath": - return {"mode": mode, "url": UPath(url, endpoint_url=endpoint_url, anon=anon)} + return {"url": UPath(url, endpoint_url=endpoint_url, anon=anon), "mode": mode} elif request.param == "use_str": return {"url": url, "mode": mode, "anon": anon, "endpoint_url": endpoint_url} From b1ecdd544823404faf37e3111e868261178aa3e5 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Sat, 14 Sep 2024 13:56:06 -0700 Subject: [PATCH 0157/1043] fix: opening a group with unspecified format finds either v2 or v3 (#2183) metadata objects --- src/zarr/api/asynchronous.py | 32 ++++++++++++++++---------------- src/zarr/core/config.py | 21 ++++++++++----------- tests/v3/test_api.py | 21 +++++++++++++++++++++ tests/v3/test_config.py | 1 + 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index d757aa2120..8a1b0c5f36 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -9,6 +9,7 @@ from zarr.core.array import Array, AsyncArray from zarr.core.common import JSON, AccessModeLiteral, ChunkCoords, MemoryOrder, ZarrFormat +from zarr.core.config import config from zarr.core.group import AsyncGroup from zarr.core.metadata.v2 import ArrayV2Metadata from zarr.core.metadata.v3 import ArrayV3Metadata @@ -126,8 +127,7 @@ def _handle_zarr_version_or_format( def _default_zarr_version() -> ZarrFormat: """return the default zarr_version""" - # TODO: set default value from config - return 3 + return cast(ZarrFormat, int(config.get("default_zarr_version", 3))) async def consolidate_metadata(*args: Any, **kwargs: Any) -> AsyncGroup: @@ -337,7 +337,10 @@ async def save_group( kwargs NumPy arrays with data to save. """ - zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + zarr_format = ( + _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + or _default_zarr_version() + ) if len(args) == 0 and len(kwargs) == 0: raise ValueError("at least one array must be provided") @@ -448,10 +451,7 @@ async def group( The new group. """ - zarr_format = ( - _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) - or _default_zarr_version() - ) + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) store_path = await make_store_path(store) if path is not None: @@ -474,7 +474,7 @@ async def group( except (KeyError, FileNotFoundError): return await AsyncGroup.create( store=store_path, - zarr_format=zarr_format, + zarr_format=zarr_format or _default_zarr_version(), exists_ok=overwrite, attributes=attributes, ) @@ -483,7 +483,7 @@ async def group( async def open_group( *, # Note: this is a change from v2 store: StoreLike | None = None, - mode: AccessModeLiteral | None = None, # not used + mode: AccessModeLiteral | None = None, cache_attrs: bool | None = None, # not used, default changed synchronizer: Any = None, # not used path: str | None = None, @@ -538,10 +538,7 @@ async def open_group( The new group. """ - zarr_format = ( - _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) - or _default_zarr_version() - ) + zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) if cache_attrs is not None: warnings.warn("cache_attrs is not yet implemented", RuntimeWarning, stacklevel=2) @@ -565,7 +562,10 @@ async def open_group( return await AsyncGroup.open(store_path, zarr_format=zarr_format) except (KeyError, FileNotFoundError): return await AsyncGroup.create( - store_path, zarr_format=zarr_format, exists_ok=True, attributes=attributes + store_path, + zarr_format=zarr_format or _default_zarr_version(), + exists_ok=True, + attributes=attributes, ) @@ -687,7 +687,7 @@ async def create( if zarr_format == 2 and chunks is None: chunks = shape - if zarr_format == 3 and chunk_shape is None: + elif zarr_format == 3 and chunk_shape is None: if chunks is not None: chunk_shape = chunks chunks = None @@ -908,7 +908,7 @@ async def open_array( if store_path.store.mode.create: return await create( store=store_path, - zarr_format=zarr_format, + zarr_format=zarr_format or _default_zarr_version(), overwrite=store_path.store.mode.overwrite, **kwargs, ) diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index 611d1faea5..45e4114389 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -28,21 +28,20 @@ def reset(self) -> None: self.refresh() -""" -The config module is responsible for managing the configuration of zarr and is based on the Donfig python library. -For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations -in the registry and then select them in the config. -e.g. an implementation of the bytes codec in a class "NewBytesCodec", requires the value of codecs.bytes.name to be -"NewBytesCodec". -Donfig can be configured programmatically, by environment variables, or from YAML files in standard locations -e.g. export ZARR_CODECS__BYTES__NAME="NewBytesCodec" -(for more information see github.com/pytroll/donfig) -Default values below point to the standard implementations of zarr-python -""" +# The config module is responsible for managing the configuration of zarr and is based on the Donfig python library. +# For selecting custom implementations of codecs, pipelines, buffers and ndbuffers, first register the implementations +# in the registry and then select them in the config. +# e.g. an implementation of the bytes codec in a class "NewBytesCodec", requires the value of codecs.bytes.name to be +# "NewBytesCodec". +# Donfig can be configured programmatically, by environment variables, or from YAML files in standard locations +# e.g. export ZARR_CODECS__BYTES__NAME="NewBytesCodec" +# (for more information see github.com/pytroll/donfig) +# Default values below point to the standard implementations of zarr-python config = Config( "zarr", defaults=[ { + "default_zarr_version": 3, "array": {"order": "C"}, "async": {"concurrency": None, "timeout": None}, "json_indent": 2, diff --git a/tests/v3/test_api.py b/tests/v3/test_api.py index 239dd1c3e2..ddfab587cc 100644 --- a/tests/v3/test_api.py +++ b/tests/v3/test_api.py @@ -8,6 +8,7 @@ from zarr import Array, Group from zarr.abc.store import Store from zarr.api.synchronous import create, load, open, open_group, save, save_array, save_group +from zarr.core.common import ZarrFormat from zarr.store.memory import MemoryStore @@ -81,6 +82,26 @@ async def test_open_group(memory_store: MemoryStore) -> None: # assert g.read_only +@pytest.mark.parametrize("zarr_format", [None, 2, 3]) +async def test_open_group_unspecified_version( + tmpdir: pathlib.Path, zarr_format: ZarrFormat +) -> None: + """regression test for https://github.com/zarr-developers/zarr-python/issues/2175""" + + # create a group with specified zarr format (could be 2, 3, or None) + _ = await zarr.api.asynchronous.open_group( + store=str(tmpdir), mode="w", zarr_format=zarr_format, attributes={"foo": "bar"} + ) + + # now open that group without specifying the format + g2 = await zarr.api.asynchronous.open_group(store=str(tmpdir), mode="r") + + assert g2.attrs == {"foo": "bar"} + + if zarr_format is not None: + assert g2.metadata.zarr_format == zarr_format + + def test_save_errors() -> None: with pytest.raises(ValueError): # no arrays provided diff --git a/tests/v3/test_config.py b/tests/v3/test_config.py index c0674ecbfd..115487ba87 100644 --- a/tests/v3/test_config.py +++ b/tests/v3/test_config.py @@ -39,6 +39,7 @@ def test_config_defaults_set() -> None: # regression test for available defaults assert config.defaults == [ { + "default_zarr_version": 3, "array": {"order": "C"}, "async": {"concurrency": None, "timeout": None}, "json_indent": 2, From fbd1658f1f95e0956a6ac294cf6a0b654841fb1c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 22:54:26 -0700 Subject: [PATCH 0158/1043] chore: update pre-commit hooks (#2188) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.4 → v0.6.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.4...v0.6.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2fd405b64d..79c68418da 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.6.5 hooks: - id: ruff args: ["--fix", "--show-fixes"] From bc82d02f312f4819453c2b7f3e8c942ec53a78c0 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 17 Sep 2024 07:02:49 -0700 Subject: [PATCH 0159/1043] fix: replace tests that went missing in #2006 (#2192) --- tests/v3/test_group.py | 154 ++++++++++++++++++++++++++++- tests/v3/test_store/test_memory.py | 50 +++++++++- 2 files changed, 201 insertions(+), 3 deletions(-) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 94b839a186..d5fb9e7b5a 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -6,6 +6,7 @@ import numpy as np import pytest +import zarr from zarr import Array, AsyncArray, AsyncGroup, Group from zarr.abc.store import Store from zarr.core.buffer import default_buffer_prototype @@ -13,7 +14,7 @@ from zarr.core.group import GroupMetadata from zarr.core.sync import sync from zarr.errors import ContainsArrayError, ContainsGroupError -from zarr.store import LocalStore, StorePath +from zarr.store import LocalStore, MemoryStore, StorePath from zarr.store.common import make_store_path from .conftest import parse_store @@ -699,3 +700,154 @@ def test_serializable_sync_group(store: LocalStore, zarr_format: ZarrFormat) -> actual = pickle.loads(p) assert actual == expected + + +async def test_group_members_async(store: LocalStore | MemoryStore) -> None: + group = AsyncGroup( + GroupMetadata(), + store_path=StorePath(store=store, path="root"), + ) + a0 = await group.create_array("a0", shape=(1,)) + g0 = await group.create_group("g0") + a1 = await g0.create_array("a1", shape=(1,)) + g1 = await g0.create_group("g1") + a2 = await g1.create_array("a2", shape=(1,)) + g2 = await g1.create_group("g2") + + # immediate children + children = sorted([x async for x in group.members()], key=lambda x: x[0]) + assert children == [ + ("a0", a0), + ("g0", g0), + ] + + nmembers = await group.nmembers() + assert nmembers == 2 + + # partial + children = sorted([x async for x in group.members(max_depth=1)], key=lambda x: x[0]) + expected = [ + ("a0", a0), + ("g0", g0), + ("g0/a1", a1), + ("g0/g1", g1), + ] + assert children == expected + nmembers = await group.nmembers(max_depth=1) + assert nmembers == 4 + + # all children + all_children = sorted([x async for x in group.members(max_depth=None)], key=lambda x: x[0]) + expected = [ + ("a0", a0), + ("g0", g0), + ("g0/a1", a1), + ("g0/g1", g1), + ("g0/g1/a2", a2), + ("g0/g1/g2", g2), + ] + assert all_children == expected + + nmembers = await group.nmembers(max_depth=None) + assert nmembers == 6 + + with pytest.raises(ValueError, match="max_depth"): + [x async for x in group.members(max_depth=-1)] + + +async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + + # create foo group + _ = await root.create_group("foo", attributes={"foo": 100}) + + # test that we can get the group using require_group + foo_group = await root.require_group("foo") + assert foo_group.attrs == {"foo": 100} + + # test that we can get the group using require_group and overwrite=True + foo_group = await root.require_group("foo", overwrite=True) + + _ = await foo_group.create_array( + "bar", shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} + ) + + # test that overwriting a group w/ children fails + # TODO: figure out why ensure_no_existing_node is not catching the foo.bar array + # + # with pytest.raises(ContainsArrayError): + # await root.require_group("foo", overwrite=True) + + # test that requiring a group where an array is fails + with pytest.raises(TypeError): + await foo_group.require_group("bar") + + +async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + # create foo group + _ = await root.create_group("foo", attributes={"foo": 100}) + # create bar group + _ = await root.create_group("bar", attributes={"bar": 200}) + + foo_group, bar_group = await root.require_groups("foo", "bar") + assert foo_group.attrs == {"foo": 100} + assert bar_group.attrs == {"bar": 200} + + # get a mix of existing and new groups + foo_group, spam_group = await root.require_groups("foo", "spam") + assert foo_group.attrs == {"foo": 100} + assert spam_group.attrs == {} + + # no names + no_group = await root.require_groups() + assert no_group == () + + +async def test_create_dataset(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + with pytest.warns(DeprecationWarning): + foo = await root.create_dataset("foo", shape=(10,), dtype="uint8") + assert foo.shape == (10,) + + with pytest.raises(ContainsArrayError), pytest.warns(DeprecationWarning): + await root.create_dataset("foo", shape=(100,), dtype="int8") + + _ = await root.create_group("bar") + with pytest.raises(ContainsGroupError), pytest.warns(DeprecationWarning): + await root.create_dataset("bar", shape=(100,), dtype="int8") + + +async def test_require_array(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: + root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + foo1 = await root.require_array("foo", shape=(10,), dtype="i8", attributes={"foo": 101}) + assert foo1.attrs == {"foo": 101} + foo2 = await root.require_array("foo", shape=(10,), dtype="i8") + assert foo2.attrs == {"foo": 101} + + # exact = False + _ = await root.require_array("foo", shape=10, dtype="f8") + + # errors w/ exact True + with pytest.raises(TypeError, match="Incompatible dtype"): + await root.require_array("foo", shape=(10,), dtype="f8", exact=True) + + with pytest.raises(TypeError, match="Incompatible shape"): + await root.require_array("foo", shape=(100, 100), dtype="i8") + + with pytest.raises(TypeError, match="Incompatible dtype"): + await root.require_array("foo", shape=(10,), dtype="f4") + + _ = await root.create_group("bar") + with pytest.raises(TypeError, match="Incompatible object"): + await root.require_array("bar", shape=(10,), dtype="int8") + + +async def test_open_mutable_mapping(): + group = await zarr.api.asynchronous.open_group(store={}, mode="w") + assert isinstance(group.store_path.store, MemoryStore) + + +def test_open_mutable_mapping_sync(): + group = zarr.open_group(store={}, mode="w") + assert isinstance(group.store_path.store, MemoryStore) diff --git a/tests/v3/test_store/test_memory.py b/tests/v3/test_store/test_memory.py index 04d17eb240..470383dfb6 100644 --- a/tests/v3/test_store/test_memory.py +++ b/tests/v3/test_store/test_memory.py @@ -4,9 +4,10 @@ import pytest -from zarr.core.buffer import Buffer, cpu -from zarr.store.memory import MemoryStore +from zarr.core.buffer import Buffer, cpu, gpu +from zarr.store.memory import GpuMemoryStore, MemoryStore from zarr.testing.store import StoreTests +from zarr.testing.utils import gpu_test class TestMemoryStore(StoreTests[MemoryStore, cpu.Buffer]): @@ -56,3 +57,48 @@ def test_serizalizable_store(self, store: MemoryStore) -> None: with pytest.raises(NotImplementedError): pickle.dumps(store) + + +@gpu_test +class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): + store_cls = GpuMemoryStore + buffer_cls = gpu.Buffer + + def set(self, store: GpuMemoryStore, key: str, value: Buffer) -> None: + store._store_dict[key] = value + + def get(self, store: MemoryStore, key: str) -> Buffer: + return store._store_dict[key] + + @pytest.fixture(scope="function", params=[None, {}]) + def store_kwargs(self, request) -> dict[str, str | None | dict[str, Buffer]]: + return {"store_dict": request.param, "mode": "r+"} + + @pytest.fixture(scope="function") + def store(self, store_kwargs: str | None | dict[str, gpu.Buffer]) -> GpuMemoryStore: + return self.store_cls(**store_kwargs) + + def test_store_repr(self, store: GpuMemoryStore) -> None: + assert str(store) == f"gpumemory://{id(store._store_dict)}" + + def test_store_supports_writes(self, store: GpuMemoryStore) -> None: + assert store.supports_writes + + def test_store_supports_listing(self, store: GpuMemoryStore) -> None: + assert store.supports_listing + + def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: + assert store.supports_partial_writes + + def test_list_prefix(self, store: GpuMemoryStore) -> None: + assert True + + def test_serizalizable_store(self, store: MemoryStore) -> None: + with pytest.raises(NotImplementedError): + store.__getstate__() + + with pytest.raises(NotImplementedError): + store.__setstate__({}) + + with pytest.raises(NotImplementedError): + pickle.dumps(store) From dd03ff0c141e0c916b35c21c6e28fc66a7b2be2e Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 17 Sep 2024 08:48:36 -0600 Subject: [PATCH 0160/1043] Silence hypothesis complex warning (#2157) * Silence hypothesis complex warning Closes #2155 * Use `array_equal` instead * Fix test * Add nan fill value test * better test * add equal_nan kwarg * cleanup * Update tests/v3/test_array.py Co-authored-by: David Stansby --------- Co-authored-by: David Stansby --- src/zarr/core/buffer/core.py | 8 ++++++-- tests/v3/test_array.py | 19 +++++++++++++++++++ tests/v3/test_properties.py | 10 ---------- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/zarr/core/buffer/core.py b/src/zarr/core/buffer/core.py index 0c6d966db9..ba629befa1 100644 --- a/src/zarr/core/buffer/core.py +++ b/src/zarr/core/buffer/core.py @@ -462,8 +462,12 @@ def __len__(self) -> int: def __repr__(self) -> str: return f"" - def all_equal(self, other: Any) -> bool: - return bool((self._data == other).all()) + def all_equal(self, other: Any, equal_nan: bool = True) -> bool: + """Compare to `other` using np.array_equal.""" + # use array_equal to obtain equal_nan=True functionality + data, other = np.broadcast_arrays(self._data, other) + result = np.array_equal(self._data, other, equal_nan=equal_nan) + return result def fill(self, value: Any) -> None: self._data.fill(value) diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index 11be51682c..b7beb63b1c 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -138,6 +138,25 @@ def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str assert arr.fill_value.dtype == arr.dtype +@pytest.mark.parametrize("store", ["memory"], indirect=True) +async def test_array_v3_nan_fill_value(store: MemoryStore) -> None: + shape = (10,) + arr = Array.create( + store=store, + shape=shape, + dtype=np.float64, + zarr_format=3, + chunk_shape=shape, + fill_value=np.nan, + ) + arr[:] = np.nan + + assert np.isnan(arr.fill_value) + assert arr.fill_value.dtype == arr.dtype + # all fill value chunk is an empty chunk, and should not be written + assert len([a async for a in store.list_prefix("/")]) == 0 + + @pytest.mark.parametrize("store", ("local",), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) async def test_serializable_async_array( diff --git a/tests/v3/test_properties.py b/tests/v3/test_properties.py index 7a085c03b7..a78e9207bd 100644 --- a/tests/v3/test_properties.py +++ b/tests/v3/test_properties.py @@ -18,11 +18,6 @@ def test_roundtrip(data: st.DataObject) -> None: @given(data=st.data()) -# The filter warning here is to silence an occasional warning in NDBuffer.all_equal -# See https://github.com/zarr-developers/zarr-python/pull/2118#issuecomment-2310280899 -# Uncomment the next line to reproduce the original failure. -# @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/ndR2z7nkDZEDADWpBL4=') -@pytest.mark.filterwarnings("ignore::RuntimeWarning") def test_basic_indexing(data: st.DataObject) -> None: zarray = data.draw(arrays()) nparray = zarray[:] @@ -37,11 +32,6 @@ def test_basic_indexing(data: st.DataObject) -> None: @given(data=st.data()) -# The filter warning here is to silence an occasional warning in NDBuffer.all_equal -# See https://github.com/zarr-developers/zarr-python/pull/2118#issuecomment-2310280899 -# Uncomment the next line to reproduce the original failure. -# @reproduce_failure('6.111.2', b'AXicY2FgZGRAB/8/eLmF7qr/C5EDADZUBRM=') -@pytest.mark.filterwarnings("ignore::RuntimeWarning") def test_vindex(data: st.DataObject) -> None: zarray = data.draw(arrays()) nparray = zarray[:] From 8c5038a752eeb6a532b49e966da0509003dc2b4c Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 17 Sep 2024 18:55:23 +0200 Subject: [PATCH 0161/1043] Narrow JSON type, ensure that `to_dict` always returns a dict, and v2 filter / compressor parsing (#2179) * fix: narrow JSON type, ensure compliance with it, and fix a variety of v2 metadata issues * remove unneeded conditional * codecpipeline no longer inherits from metadata, ditches to_dict and from_dict methods * rename from_list to from_codecs --- src/zarr/abc/codec.py | 18 ++------- src/zarr/abc/metadata.py | 2 +- src/zarr/codecs/_v2.py | 11 +++--- src/zarr/codecs/blosc.py | 4 +- src/zarr/codecs/bytes.py | 2 +- src/zarr/codecs/pipeline.py | 30 +++------------ src/zarr/codecs/sharding.py | 20 +++++----- src/zarr/codecs/transpose.py | 2 +- src/zarr/core/array.py | 20 +++------- src/zarr/core/common.py | 6 +-- src/zarr/core/metadata/v2.py | 62 ++++++++++++++++++++----------- src/zarr/core/metadata/v3.py | 51 ++++++++++++------------- tests/v3/test_metadata/test_v2.py | 8 ++-- 13 files changed, 106 insertions(+), 130 deletions(-) diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index cc32b9bcfc..2098d989e9 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -17,7 +17,6 @@ from zarr.abc.store import ByteGetter, ByteSetter from zarr.core.array_spec import ArraySpec from zarr.core.chunk_grids import ChunkGrid - from zarr.core.common import JSON from zarr.core.indexing import SelectorTuple __all__ = [ @@ -242,7 +241,7 @@ async def encode_partial( ) -class CodecPipeline(Metadata): +class CodecPipeline: """Base class for implementing CodecPipeline. A CodecPipeline implements the read and write paths for chunk data. On the read path, it is responsible for fetching chunks from a store (via ByteGetter), @@ -266,12 +265,12 @@ def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: @classmethod @abstractmethod - def from_list(cls, codecs: Iterable[Codec]) -> Self: - """Creates a codec pipeline from a list of codecs. + def from_codecs(cls, codecs: Iterable[Codec]) -> Self: + """Creates a codec pipeline from an iterable of codecs. Parameters ---------- - codecs : list[Codec] + codecs : Iterable[Codec] Returns ------- @@ -402,15 +401,6 @@ async def write( """ ... - @classmethod - def from_dict(cls, data: Iterable[JSON | Codec]) -> Self: - """ - Create an instance of the model from a dictionary - """ - ... - - return cls(**data) - async def _batching_helper( func: Callable[[CodecInput, ArraySpec], Awaitable[CodecOutput | None]], diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py index d9b11af883..7ea668c891 100644 --- a/src/zarr/abc/metadata.py +++ b/src/zarr/abc/metadata.py @@ -15,7 +15,7 @@ @dataclass(frozen=True) class Metadata: - def to_dict(self) -> JSON: + def to_dict(self) -> dict[str, JSON]: """ Recursively serialize this model to a dictionary. This method inspects the fields of self and calls `x.to_dict()` for any fields that diff --git a/src/zarr/codecs/_v2.py b/src/zarr/codecs/_v2.py index eb8ec435f5..c8bc558349 100644 --- a/src/zarr/codecs/_v2.py +++ b/src/zarr/codecs/_v2.py @@ -67,7 +67,7 @@ def compute_encoded_size(self, _input_byte_length: int, _chunk_spec: ArraySpec) @dataclass(frozen=True) class V2Filters(ArrayArrayCodec): - filters: list[dict[str, JSON]] + filters: tuple[numcodecs.abc.Codec, ...] | None is_fixed_size = False @@ -79,8 +79,7 @@ async def _decode_single( chunk_ndarray = chunk_array.as_ndarray_like() # apply filters in reverse order if self.filters is not None: - for filter_metadata in self.filters[::-1]: - filter = numcodecs.get_codec(filter_metadata) + for filter in self.filters[::-1]: chunk_ndarray = await to_thread(filter.decode, chunk_ndarray) # ensure correct chunk shape @@ -99,9 +98,9 @@ async def _encode_single( ) -> NDBuffer | None: chunk_ndarray = chunk_array.as_ndarray_like().ravel(order=chunk_spec.order) - for filter_metadata in self.filters: - filter = numcodecs.get_codec(filter_metadata) - chunk_ndarray = await to_thread(filter.encode, chunk_ndarray) + if self.filters is not None: + for filter in self.filters: + chunk_ndarray = await to_thread(filter.encode, chunk_ndarray) return get_ndbuffer_class().from_ndarray_like(chunk_ndarray) diff --git a/src/zarr/codecs/blosc.py b/src/zarr/codecs/blosc.py index f831dc960d..7b10d91a6a 100644 --- a/src/zarr/codecs/blosc.py +++ b/src/zarr/codecs/blosc.py @@ -127,9 +127,9 @@ def to_dict(self) -> dict[str, JSON]: "name": "blosc", "configuration": { "typesize": self.typesize, - "cname": self.cname, + "cname": self.cname.value, "clevel": self.clevel, - "shuffle": self.shuffle, + "shuffle": self.shuffle.value, "blocksize": self.blocksize, }, } diff --git a/src/zarr/codecs/bytes.py b/src/zarr/codecs/bytes.py index bc3207be2e..7a683411e9 100644 --- a/src/zarr/codecs/bytes.py +++ b/src/zarr/codecs/bytes.py @@ -53,7 +53,7 @@ def to_dict(self) -> dict[str, JSON]: if self.endian is None: return {"name": "bytes"} else: - return {"name": "bytes", "configuration": {"endian": self.endian}} + return {"name": "bytes", "configuration": {"endian": self.endian.value}} def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: if array_spec.dtype.itemsize == 0: diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/codecs/pipeline.py index 3a400d6eaf..182621c59f 100644 --- a/src/zarr/codecs/pipeline.py +++ b/src/zarr/codecs/pipeline.py @@ -1,6 +1,5 @@ from __future__ import annotations -from collections.abc import Iterable, Iterator from dataclasses import dataclass from itertools import islice, pairwise from typing import TYPE_CHECKING, Any, TypeVar @@ -15,12 +14,14 @@ Codec, CodecPipeline, ) -from zarr.core.common import JSON, ChunkCoords, concurrent_map, parse_named_configuration +from zarr.core.common import ChunkCoords, concurrent_map from zarr.core.config import config from zarr.core.indexing import SelectorTuple, is_scalar, is_total_slice -from zarr.registry import get_codec_class, register_pipeline +from zarr.registry import register_pipeline if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + import numpy as np from typing_extensions import Self @@ -68,30 +69,11 @@ class BatchedCodecPipeline(CodecPipeline): bytes_bytes_codecs: tuple[BytesBytesCodec, ...] batch_size: int - @classmethod - def from_dict(cls, data: Iterable[JSON | Codec], *, batch_size: int | None = None) -> Self: - out: list[Codec] = [] - if not isinstance(data, Iterable): - raise TypeError(f"Expected iterable, got {type(data)}") - - for c in data: - if isinstance( - c, ArrayArrayCodec | ArrayBytesCodec | BytesBytesCodec - ): # Can't use Codec here because of mypy limitation - out.append(c) - else: - name_parsed, _ = parse_named_configuration(c, require_configuration=False) - out.append(get_codec_class(name_parsed).from_dict(c)) # type: ignore[arg-type] - return cls.from_list(out, batch_size=batch_size) - - def to_dict(self) -> JSON: - return [c.to_dict() for c in self] - def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self: - return type(self).from_list([c.evolve_from_array_spec(array_spec=array_spec) for c in self]) + return type(self).from_codecs(c.evolve_from_array_spec(array_spec=array_spec) for c in self) @classmethod - def from_list(cls, codecs: Iterable[Codec], *, batch_size: int | None = None) -> Self: + def from_codecs(cls, codecs: Iterable[Codec], *, batch_size: int | None = None) -> Self: array_array_codecs, array_bytes_codec, bytes_bytes_codecs = codecs_from_list(codecs) return cls( diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index df7f5978a7..3ae51ce54b 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -68,7 +68,7 @@ class ShardingCodecIndexLocation(Enum): end = "end" -def parse_index_location(data: JSON) -> ShardingCodecIndexLocation: +def parse_index_location(data: object) -> ShardingCodecIndexLocation: return parse_enum(data, ShardingCodecIndexLocation) @@ -333,7 +333,7 @@ def __init__( chunk_shape: ChunkCoordsLike, codecs: Iterable[Codec | dict[str, JSON]] = (BytesCodec(),), index_codecs: Iterable[Codec | dict[str, JSON]] = (BytesCodec(), Crc32cCodec()), - index_location: ShardingCodecIndexLocation = ShardingCodecIndexLocation.end, + index_location: ShardingCodecIndexLocation | str = ShardingCodecIndexLocation.end, ) -> None: chunk_shape_parsed = parse_shapelike(chunk_shape) codecs_parsed = parse_codecs(codecs) @@ -373,16 +373,16 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: @property def codec_pipeline(self) -> CodecPipeline: - return get_pipeline_class().from_list(self.codecs) + return get_pipeline_class().from_codecs(self.codecs) def to_dict(self) -> dict[str, JSON]: return { "name": "sharding_indexed", "configuration": { - "chunk_shape": list(self.chunk_shape), - "codecs": [s.to_dict() for s in self.codecs], - "index_codecs": [s.to_dict() for s in self.index_codecs], - "index_location": self.index_location, + "chunk_shape": self.chunk_shape, + "codecs": tuple([s.to_dict() for s in self.codecs]), + "index_codecs": tuple([s.to_dict() for s in self.index_codecs]), + "index_location": self.index_location.value, }, } @@ -620,7 +620,7 @@ async def _decode_shard_index( index_array = next( iter( await get_pipeline_class() - .from_list(self.index_codecs) + .from_codecs(self.index_codecs) .decode( [(index_bytes, self._get_index_chunk_spec(chunks_per_shard))], ) @@ -633,7 +633,7 @@ async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: index_bytes = next( iter( await get_pipeline_class() - .from_list(self.index_codecs) + .from_codecs(self.index_codecs) .encode( [ ( @@ -651,7 +651,7 @@ async def _encode_shard_index(self, index: _ShardIndex) -> Buffer: def _shard_index_size(self, chunks_per_shard: ChunkCoords) -> int: return ( get_pipeline_class() - .from_list(self.index_codecs) + .from_codecs(self.index_codecs) .compute_encoded_size( 16 * product(chunks_per_shard), self._get_index_chunk_spec(chunks_per_shard) ) diff --git a/src/zarr/codecs/transpose.py b/src/zarr/codecs/transpose.py index 9bb795a3a1..45eb5bbe5f 100644 --- a/src/zarr/codecs/transpose.py +++ b/src/zarr/codecs/transpose.py @@ -45,7 +45,7 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: return cls(**configuration_parsed) # type: ignore[arg-type] def to_dict(self) -> dict[str, JSON]: - return {"name": "transpose", "configuration": {"order": list(self.order)}} + return {"name": "transpose", "configuration": {"order": tuple(self.order)}} def validate(self, shape: tuple[int, ...], dtype: np.dtype[Any], chunk_grid: ChunkGrid) -> None: if len(self.order) != len(shape): diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 7311b6eec2..3a455b239f 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -87,10 +87,10 @@ def parse_array_metadata(data: Any) -> ArrayV2Metadata | ArrayV3Metadata: def create_codec_pipeline(metadata: ArrayV2Metadata | ArrayV3Metadata) -> CodecPipeline: if isinstance(metadata, ArrayV3Metadata): - return get_pipeline_class().from_list(metadata.codecs) + return get_pipeline_class().from_codecs(metadata.codecs) elif isinstance(metadata, ArrayV2Metadata): - return get_pipeline_class().from_list( - [V2Filters(metadata.filters or []), V2Compressor(metadata.compressor)] + return get_pipeline_class().from_codecs( + [V2Filters(metadata.filters), V2Compressor(metadata.compressor)] ) else: raise TypeError @@ -299,8 +299,6 @@ async def _create_v2( attributes: dict[str, JSON] | None = None, exists_ok: bool = False, ) -> AsyncArray: - import numcodecs - if not exists_ok: await ensure_no_existing_node(store_path, zarr_format=2) if order is None: @@ -315,15 +313,9 @@ async def _create_v2( chunks=chunks, order=order, dimension_separator=dimension_separator, - fill_value=0 if fill_value is None else fill_value, - compressor=( - numcodecs.get_codec(compressor).get_config() if compressor is not None else None - ), - filters=( - [numcodecs.get_codec(filter).get_config() for filter in filters] - if filters is not None - else None - ), + fill_value=fill_value, + compressor=compressor, + filters=filters, attributes=attributes, ) array = cls(metadata=metadata, store_path=store_path) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 99ab58fae9..906467005f 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -4,7 +4,7 @@ import contextvars import functools import operator -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from enum import Enum from typing import ( TYPE_CHECKING, @@ -32,7 +32,7 @@ ChunkCoords = tuple[int, ...] ChunkCoordsLike = Iterable[int] ZarrFormat = Literal[2, 3] -JSON = None | str | int | float | Enum | dict[str, "JSON"] | list["JSON"] | tuple["JSON", ...] +JSON = None | str | int | float | Mapping[str, "JSON"] | tuple["JSON", ...] MemoryOrder = Literal["C", "F"] AccessModeLiteral = Literal["r", "r+", "a", "w", "w-"] @@ -80,7 +80,7 @@ def enum_names(enum: type[E]) -> Iterator[str]: yield item.name -def parse_enum(data: JSON, cls: type[E]) -> E: +def parse_enum(data: object, cls: type[E]) -> E: if isinstance(data, cls): return data if not isinstance(data, str): diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index 6d5ecd7e86..af7821bea7 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import Iterable from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -14,6 +15,7 @@ import json from dataclasses import dataclass, field, replace +import numcodecs import numpy as np from zarr.core.array_spec import ArraySpec @@ -31,9 +33,9 @@ class ArrayV2Metadata(ArrayMetadata): data_type: np.dtype[Any] fill_value: None | int | float = 0 order: Literal["C", "F"] = "C" - filters: list[dict[str, JSON]] | None = None + filters: tuple[numcodecs.abc.Codec, ...] | None = None dimension_separator: Literal[".", "/"] = "." - compressor: dict[str, JSON] | None = None + compressor: numcodecs.abc.Codec | None = None attributes: dict[str, JSON] = field(default_factory=dict) zarr_format: Literal[2] = field(init=False, default=2) @@ -46,8 +48,8 @@ def __init__( fill_value: Any, order: Literal["C", "F"], dimension_separator: Literal[".", "/"] = ".", - compressor: dict[str, JSON] | None = None, - filters: list[dict[str, JSON]] | None = None, + compressor: numcodecs.abc.Codec | dict[str, JSON] | None = None, + filters: Iterable[numcodecs.abc.Codec | dict[str, JSON]] | None = None, attributes: dict[str, JSON] | None = None, ): """ @@ -104,11 +106,6 @@ def _json_convert( raise TypeError zarray_dict = self.to_dict() - - # todo: remove this check when we can ensure that to_dict always returns dicts. - if not isinstance(zarray_dict, dict): - raise TypeError(f"Invalid type: got {type(zarray_dict)}, expected dict.") - zattrs_dict = zarray_dict.pop("attributes", {}) json_indent = config.get("json_indent") return { @@ -128,13 +125,8 @@ def from_dict(cls, data: dict[str, Any]) -> ArrayV2Metadata: _ = parse_zarr_format(_data.pop("zarr_format")) return cls(**_data) - def to_dict(self) -> JSON: + def to_dict(self) -> dict[str, JSON]: zarray_dict = super().to_dict() - - # todo: remove this check when we can ensure that to_dict always returns dicts. - if not isinstance(zarray_dict, dict): - raise TypeError(f"Invalid type: got {type(zarray_dict)}, expected dict.") - _ = zarray_dict.pop("chunk_grid") zarray_dict["chunks"] = self.chunk_grid.chunk_shape @@ -165,18 +157,44 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) -def parse_zarr_format(data: Literal[2]) -> Literal[2]: +def parse_zarr_format(data: object) -> Literal[2]: if data == 2: - return data + return 2 raise ValueError(f"Invalid value. Expected 2. Got {data}.") -def parse_filters(data: list[dict[str, JSON]] | None) -> list[dict[str, JSON]] | None: - return data +def parse_filters(data: object) -> tuple[numcodecs.abc.Codec, ...] | None: + """ + Parse a potential tuple of filters + """ + out: list[numcodecs.abc.Codec] = [] + if data is None: + return data + if isinstance(data, Iterable): + for idx, val in enumerate(data): + if isinstance(val, numcodecs.abc.Codec): + out.append(val) + elif isinstance(val, dict): + out.append(numcodecs.get_codec(val)) + else: + msg = f"Invalid filter at index {idx}. Expected a numcodecs.abc.Codec or a dict representation of numcodecs.abc.Codec. Got {type(val)} instead." + raise TypeError(msg) + return tuple(out) + msg = f"Invalid filters. Expected None, an iterable of numcodecs.abc.Codec or dict representations of numcodecs.abc.Codec. Got {type(data)} instead." + raise TypeError(msg) -def parse_compressor(data: dict[str, JSON] | None) -> dict[str, JSON] | None: - return data + +def parse_compressor(data: object) -> numcodecs.abc.Codec | None: + """ + Parse a potential compressor. + """ + if data is None or isinstance(data, numcodecs.abc.Codec): + return data + if isinstance(data, dict): + return numcodecs.get_codec(data) + msg = f"Invalid compressor. Expected None, a numcodecs.abc.Codec, or a dict representation of a numcodecs.abc.Codec. Got {type(data)} instead." + raise ValueError(msg) def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: @@ -189,7 +207,7 @@ def parse_metadata(data: ArrayV2Metadata) -> ArrayV2Metadata: return data -def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: +def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: """ Parse a potential fill value into a value that is compatible with the provided dtype. diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 195c3bd0a2..10047cbb93 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -19,7 +19,7 @@ import numcodecs.abc import numpy as np -from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec, CodecPipeline +from zarr.abc.codec import ArrayArrayCodec, ArrayBytesCodec, BytesBytesCodec, Codec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid @@ -27,22 +27,22 @@ from zarr.core.common import ZARR_JSON, parse_dtype, parse_named_configuration, parse_shapelike from zarr.core.config import config from zarr.core.metadata.common import ArrayMetadata, parse_attributes -from zarr.registry import get_codec_class, get_pipeline_class +from zarr.registry import get_codec_class -def parse_zarr_format(data: Literal[3]) -> Literal[3]: +def parse_zarr_format(data: object) -> Literal[3]: if data == 3: - return data + return 3 raise ValueError(f"Invalid value. Expected 3. Got {data}.") -def parse_node_type_array(data: Literal["array"]) -> Literal["array"]: +def parse_node_type_array(data: object) -> Literal["array"]: if data == "array": - return data + return "array" raise ValueError(f"Invalid value. Expected 'array'. Got {data}.") -def parse_codecs(data: Iterable[Codec | dict[str, JSON]]) -> tuple[Codec, ...]: +def parse_codecs(data: object) -> tuple[Codec, ...]: out: tuple[Codec, ...] = () if not isinstance(data, Iterable): @@ -60,10 +60,10 @@ def parse_codecs(data: Iterable[Codec | dict[str, JSON]]) -> tuple[Codec, ...]: return out -def parse_dimension_names(data: None | Iterable[str | None]) -> tuple[str | None, ...] | None: +def parse_dimension_names(data: object) -> tuple[str | None, ...] | None: if data is None: return data - elif all(isinstance(x, type(None) | str) for x in data): + elif isinstance(data, Iterable) and all(isinstance(x, type(None) | str) for x in data): return tuple(data) else: msg = f"Expected either None or a iterable of str, got {type(data)}" @@ -169,7 +169,7 @@ def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: return self.chunk_key_encoding.encode_chunk_key(chunk_coords) def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - def _json_convert(o: Any) -> Any: + def _json_convert(o: object) -> Any: if isinstance(o, np.dtype): return str(o) if np.isscalar(o): @@ -206,14 +206,14 @@ def _json_convert(o: Any) -> Any: } @classmethod - def from_dict(cls, data: dict[str, JSON]) -> ArrayV3Metadata: + def from_dict(cls, data: dict[str, JSON]) -> Self: # make a copy because we are modifying the dict _data = data.copy() - # TODO: Remove the type: ignores[] comments below and use a TypedDict to type `data` + # check that the zarr_format attribute is correct - _ = parse_zarr_format(_data.pop("zarr_format")) # type: ignore[arg-type] + _ = parse_zarr_format(_data.pop("zarr_format")) # check that the node_type attribute is correct - _ = parse_node_type_array(_data.pop("node_type")) # type: ignore[arg-type] + _ = parse_node_type_array(_data.pop("node_type")) # dimension_names key is optional, normalize missing to `None` _data["dimension_names"] = _data.pop("dimension_names", None) @@ -221,7 +221,7 @@ def from_dict(cls, data: dict[str, JSON]) -> ArrayV3Metadata: _data["attributes"] = _data.pop("attributes", None) return cls(**_data) # type: ignore[arg-type] - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> dict[str, JSON]: out_dict = super().to_dict() if not isinstance(out_dict, dict): @@ -240,12 +240,6 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) -def create_pipeline(data: Iterable[Codec | JSON]) -> CodecPipeline: - if not isinstance(data, Iterable): - raise TypeError(f"Expected iterable, got {type(data)}") - return get_pipeline_class().from_dict(data) - - BOOL = np.bool_ BOOL_DTYPE = np.dtypes.BoolDType INTEGER_DTYPE = ( @@ -266,23 +260,23 @@ def create_pipeline(data: Iterable[Codec | JSON]) -> CodecPipeline: @overload -def parse_fill_value(fill_value: Any, dtype: BOOL_DTYPE) -> BOOL: ... +def parse_fill_value(fill_value: object, dtype: BOOL_DTYPE) -> BOOL: ... @overload -def parse_fill_value(fill_value: Any, dtype: INTEGER_DTYPE) -> INTEGER: ... +def parse_fill_value(fill_value: object, dtype: INTEGER_DTYPE) -> INTEGER: ... @overload -def parse_fill_value(fill_value: Any, dtype: FLOAT_DTYPE) -> FLOAT: ... +def parse_fill_value(fill_value: object, dtype: FLOAT_DTYPE) -> FLOAT: ... @overload -def parse_fill_value(fill_value: Any, dtype: COMPLEX_DTYPE) -> COMPLEX: ... +def parse_fill_value(fill_value: object, dtype: COMPLEX_DTYPE) -> COMPLEX: ... @overload -def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: +def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: # This dtype[Any] is unfortunately necessary right now. # See https://github.com/zarr-developers/zarr-python/issues/2131#issuecomment-2318010899 # for more details, but `dtype` here (which comes from `parse_dtype`) @@ -294,7 +288,8 @@ def parse_fill_value(fill_value: Any, dtype: np.dtype[Any]) -> Any: def parse_fill_value( - fill_value: Any, dtype: BOOL_DTYPE | INTEGER_DTYPE | FLOAT_DTYPE | COMPLEX_DTYPE | np.dtype[Any] + fill_value: object, + dtype: BOOL_DTYPE | INTEGER_DTYPE | FLOAT_DTYPE | COMPLEX_DTYPE | np.dtype[Any], ) -> BOOL | INTEGER | FLOAT | COMPLEX | Any: """ Parse `fill_value`, a potential fill value, into an instance of `dtype`, a data type. @@ -333,7 +328,7 @@ def parse_fill_value( raise ValueError(msg) msg = f"Cannot parse non-string sequence {fill_value} as a scalar with type {dtype}." raise TypeError(msg) - return dtype.type(fill_value) + return dtype.type(fill_value) # type: ignore[arg-type] # For type checking diff --git a/tests/v3/test_metadata/test_v2.py b/tests/v3/test_metadata/test_v2.py index 4465a86471..3ea702eecd 100644 --- a/tests/v3/test_metadata/test_v2.py +++ b/tests/v3/test_metadata/test_v2.py @@ -9,9 +9,9 @@ from zarr.abc.codec import Codec +import numcodecs import pytest -from zarr.codecs import GzipCodec from zarr.core.metadata.v2 import parse_zarr_format @@ -26,14 +26,14 @@ def test_parse_zarr_format_invalid(data: Any) -> None: @pytest.mark.parametrize("attributes", [None, {"foo": "bar"}]) -@pytest.mark.parametrize("filters", [(), (GzipCodec().to_dict())]) -@pytest.mark.parametrize("compressor", [None, GzipCodec().to_dict()]) +@pytest.mark.parametrize("filters", [None, (), (numcodecs.GZip(),)]) +@pytest.mark.parametrize("compressor", [None, numcodecs.GZip()]) @pytest.mark.parametrize("fill_value", [0, 1]) @pytest.mark.parametrize("order", ["C", "F"]) @pytest.mark.parametrize("dimension_separator", [".", "/", None]) def test_metadata_to_dict( compressor: Codec | None, - filters: list[Codec] | None, + filters: tuple[Codec] | None, fill_value: Any, order: Literal["C", "F"], dimension_separator: Literal[".", "/"] | None, From 0903dd6409dbada9325f1751906fa016e1b56287 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 19 Sep 2024 10:53:04 -0500 Subject: [PATCH 0162/1043] Fixed codec for v2 data with no fill value (#2207) xref https://github.com/zarr-developers/zarr-python/pull/2179/files#r1765600769 --- src/zarr/core/array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 3a455b239f..2cf6d69404 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -313,7 +313,7 @@ async def _create_v2( chunks=chunks, order=order, dimension_separator=dimension_separator, - fill_value=fill_value, + fill_value=0 if fill_value is None else fill_value, compressor=compressor, filters=filters, attributes=attributes, From 06e3215faab96a8092e7874958f2b700fa362969 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 19 Sep 2024 18:26:23 +0200 Subject: [PATCH 0163/1043] implement `store.list_prefix` and `store._set_many` (#2064) * implement store.list_prefix and store._set_dict * simplify string handling * use asyncio.gather in _set_dict * add docstrings to list_prefix methods, and make invocation of _find more explicit * add byterangerequest type * fix: activate test of localstore.list_prefix, fix zipstore.list_prefix --- src/zarr/abc/store.py | 14 +++- src/zarr/core/common.py | 1 + src/zarr/core/sync.py | 17 ++++ src/zarr/store/local.py | 12 ++- src/zarr/store/memory.py | 14 +++- src/zarr/store/remote.py | 18 ++++- src/zarr/store/zip.py | 14 +++- src/zarr/testing/store.py | 123 ++++++++++++++--------------- tests/v3/test_store/test_local.py | 3 - tests/v3/test_store/test_remote.py | 24 +++--- 10 files changed, 150 insertions(+), 90 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 95d55a2ce0..f95ba34efd 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator +from asyncio import gather +from collections.abc import AsyncGenerator, Iterable from typing import Any, NamedTuple, Protocol, runtime_checkable from typing_extensions import Self @@ -158,6 +159,13 @@ async def set(self, key: str, value: Buffer) -> None: """ ... + async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: + """ + Insert multiple (key, value) pairs into storage. + """ + await gather(*(self.set(key, value) for key, value in values)) + return None + @property @abstractmethod def supports_deletes(self) -> bool: @@ -211,7 +219,9 @@ def list(self) -> AsyncGenerator[str, None]: @abstractmethod def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: - """Retrieve all keys in the store with a given prefix. + """ + Retrieve all keys in the store that begin with a given prefix. Keys are returned with the + common leading prefix removed. Parameters ---------- diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 906467005f..8ebe5160bd 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -27,6 +27,7 @@ ZGROUP_JSON = ".zgroup" ZATTRS_JSON = ".zattrs" +ByteRangeRequest = tuple[int | None, int | None] BytesLike = bytes | bytearray | memoryview ShapeLike = tuple[int, ...] | int ChunkCoords = tuple[int, ...] diff --git a/src/zarr/core/sync.py b/src/zarr/core/sync.py index ff7f9a43af..db3dce79b2 100644 --- a/src/zarr/core/sync.py +++ b/src/zarr/core/sync.py @@ -113,6 +113,23 @@ def _get_loop() -> asyncio.AbstractEventLoop: return loop[0] +async def _collect_aiterator(data: AsyncIterator[T]) -> tuple[T, ...]: + """ + Collect an entire async iterator into a tuple + """ + result = [] + async for x in data: + result.append(x) + return tuple(result) + + +def collect_aiterator(data: AsyncIterator[T]) -> tuple[T, ...]: + """ + Synchronously collect an entire async iterator into a tuple. + """ + return sync(_collect_aiterator(data)) + + class SyncMixin: def _sync(self, coroutine: Coroutine[Any, Any, T]) -> T: # TODO: refactor this to to take *args and **kwargs and pass those to the method diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index 5fd48c2db0..c78837586f 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -191,7 +191,9 @@ async def list(self) -> AsyncGenerator[str, None]: yield str(p).replace(to_strip, "") async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: - """Retrieve all keys in the store with a given prefix. + """ + Retrieve all keys in the store that begin with a given prefix. Keys are returned with the + common leading prefix removed. Parameters ---------- @@ -201,14 +203,10 @@ async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: ------- AsyncGenerator[str, None] """ + to_strip = os.path.join(str(self.root / prefix)) for p in (self.root / prefix).rglob("*"): if p.is_file(): - yield str(p) - - to_strip = str(self.root) + "/" - for p in (self.root / prefix).rglob("*"): - if p.is_file(): - yield str(p).replace(to_strip, "") + yield str(p.relative_to(to_strip)) async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: """ diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index 13e289f374..e304419768 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -124,9 +124,21 @@ async def list(self) -> AsyncGenerator[str, None]: async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: for key in self._store_dict: if key.startswith(prefix): - yield key + yield key.removeprefix(prefix) async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + """ + Retrieve all keys in the store that begin with a given prefix. Keys are returned with the + common leading prefix removed. + + Parameters + ---------- + prefix : str + + Returns + ------- + AsyncGenerator[str, None] + """ if prefix.endswith("/"): prefix = prefix[:-1] diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index e3e2ba3447..084ef986b1 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -216,5 +216,19 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: yield onefile.removeprefix(self.path).removeprefix("/") async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: - for onefile in await self._fs._ls(prefix, detail=False): - yield onefile + """ + Retrieve all keys in the store that begin with a given prefix. Keys are returned with the + common leading prefix removed. + + Parameters + ---------- + prefix : str + + Returns + ------- + AsyncGenerator[str, None] + """ + + find_str = "/".join([self.path, prefix]) + for onefile in await self._fs._find(find_str, detail=False, maxdepth=None, withdirs=False): + yield onefile.removeprefix(find_str) diff --git a/src/zarr/store/zip.py b/src/zarr/store/zip.py index ea31ad934a..2e4927aced 100644 --- a/src/zarr/store/zip.py +++ b/src/zarr/store/zip.py @@ -209,9 +209,21 @@ async def list(self) -> AsyncGenerator[str, None]: yield key async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + """ + Retrieve all keys in the store that begin with a given prefix. Keys are returned with the + common leading prefix removed. + + Parameters + ---------- + prefix : str + + Returns + ------- + AsyncGenerator[str, None] + """ async for key in self.list(): if key.startswith(prefix): - yield key + yield key.removeprefix(prefix) async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: if prefix.endswith("/"): diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index a08b6960db..8a9f27e4b8 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -3,9 +3,9 @@ import pytest -import zarr.api.asynchronous from zarr.abc.store import AccessMode, Store from zarr.core.buffer import Buffer, default_buffer_prototype +from zarr.core.sync import _collect_aiterator from zarr.store._utils import _normalize_interval_index from zarr.testing.utils import assert_bytes_equal @@ -123,6 +123,18 @@ async def test_set(self, store: S, key: str, data: bytes) -> None: observed = self.get(store, key) assert_bytes_equal(observed, data_buf) + async def test_set_many(self, store: S) -> None: + """ + Test that a dict of key : value pairs can be inserted into the store via the + `_set_many` method. + """ + keys = ["zarr.json", "c/0", "foo/c/0.0", "foo/0/0"] + data_buf = [self.buffer_cls.from_bytes(k.encode()) for k in keys] + store_dict = dict(zip(keys, data_buf, strict=True)) + await store._set_many(store_dict.items()) + for k, v in store_dict.items(): + assert self.get(store, k).to_bytes() == v.to_bytes() + @pytest.mark.parametrize( "key_ranges", ( @@ -185,76 +197,57 @@ async def test_clear(self, store: S) -> None: assert await store.empty() async def test_list(self, store: S) -> None: - assert [k async for k in store.list()] == [] - await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) - keys = [k async for k in store.list()] - assert keys == ["foo/zarr.json"], keys - - expected = ["foo/zarr.json"] - for i in range(10): - key = f"foo/c/{i}" - expected.append(key) - await store.set( - f"foo/c/{i}", self.buffer_cls.from_bytes(i.to_bytes(length=3, byteorder="little")) - ) + assert await _collect_aiterator(store.list()) == () + prefix = "foo" + data = self.buffer_cls.from_bytes(b"") + store_dict = { + prefix + "/zarr.json": data, + **{prefix + f"/c/{idx}": data for idx in range(10)}, + } + await store._set_many(store_dict.items()) + expected_sorted = sorted(store_dict.keys()) + observed = await _collect_aiterator(store.list()) + observed_sorted = sorted(observed) + assert observed_sorted == expected_sorted - @pytest.mark.xfail async def test_list_prefix(self, store: S) -> None: - # TODO: we currently don't use list_prefix anywhere - raise NotImplementedError + """ + Test that the `list_prefix` method works as intended. Given a prefix, it should return + all the keys in storage that start with this prefix. Keys should be returned with the shared + prefix removed. + """ + prefixes = ("", "a/", "a/b/", "a/b/c/") + data = self.buffer_cls.from_bytes(b"") + fname = "zarr.json" + store_dict = {p + fname: data for p in prefixes} + + await store._set_many(store_dict.items()) + + for prefix in prefixes: + observed = tuple(sorted(await _collect_aiterator(store.list_prefix(prefix)))) + expected: tuple[str, ...] = () + for key in store_dict.keys(): + if key.startswith(prefix): + expected += (key.removeprefix(prefix),) + expected = tuple(sorted(expected)) + assert observed == expected async def test_list_dir(self, store: S) -> None: - out = [k async for k in store.list_dir("")] - assert out == [] - assert [k async for k in store.list_dir("foo")] == [] - await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) - await store.set("group-0/zarr.json", self.buffer_cls.from_bytes(b"\x01")) # group - await store.set("group-0/group-1/zarr.json", self.buffer_cls.from_bytes(b"\x01")) # group - await store.set("group-0/group-1/a1/zarr.json", self.buffer_cls.from_bytes(b"\x01")) - await store.set("group-0/group-1/a2/zarr.json", self.buffer_cls.from_bytes(b"\x01")) - await store.set("group-0/group-1/a3/zarr.json", self.buffer_cls.from_bytes(b"\x01")) - - keys_expected = ["foo", "group-0"] - keys_observed = [k async for k in store.list_dir("")] - assert set(keys_observed) == set(keys_expected) - - keys_expected = ["zarr.json"] - keys_observed = [k async for k in store.list_dir("foo")] - - assert len(keys_observed) == len(keys_expected), keys_observed - assert set(keys_observed) == set(keys_expected), keys_observed - - keys_observed = [k async for k in store.list_dir("foo/")] - assert len(keys_expected) == len(keys_observed), keys_observed - assert set(keys_observed) == set(keys_expected), keys_observed - - keys_observed = [k async for k in store.list_dir("group-0")] - keys_expected = ["zarr.json", "group-1"] - - assert len(keys_observed) == len(keys_expected), keys_observed - assert set(keys_observed) == set(keys_expected), keys_observed - - keys_observed = [k async for k in store.list_dir("group-0/")] - assert len(keys_expected) == len(keys_observed), keys_observed - assert set(keys_observed) == set(keys_expected), keys_observed + root = "foo" + store_dict = { + root + "/zarr.json": self.buffer_cls.from_bytes(b"bar"), + root + "/c/1": self.buffer_cls.from_bytes(b"\x01"), + } - keys_observed = [k async for k in store.list_dir("group-0/group-1")] - keys_expected = ["zarr.json", "a1", "a2", "a3"] + assert await _collect_aiterator(store.list_dir("")) == () + assert await _collect_aiterator(store.list_dir(root)) == () - assert len(keys_observed) == len(keys_expected), keys_observed - assert set(keys_observed) == set(keys_expected), keys_observed + await store._set_many(store_dict.items()) - keys_observed = [k async for k in store.list_dir("group-0/group-1")] - assert len(keys_expected) == len(keys_observed), keys_observed - assert set(keys_observed) == set(keys_expected), keys_observed + keys_observed = await _collect_aiterator(store.list_dir(root)) + keys_expected = {k.removeprefix(root + "/").split("/")[0] for k in store_dict.keys()} - async def test_set_get(self, store_kwargs: dict[str, Any]) -> None: - kwargs = {**store_kwargs, **{"mode": "w"}} - store = self.store_cls(**kwargs) - await zarr.api.asynchronous.open_array(store=store, path="a", mode="w", shape=(4,)) - keys = [x async for x in store.list()] - assert keys == ["a/zarr.json"] + assert sorted(keys_observed) == sorted(keys_expected) - # no errors - await zarr.api.asynchronous.open_array(store=store, path="a", mode="r") - await zarr.api.asynchronous.open_array(store=store, path="a", mode="a") + keys_observed = await _collect_aiterator(store.list_dir(root + "/")) + assert sorted(keys_expected) == sorted(keys_observed) diff --git a/tests/v3/test_store/test_local.py b/tests/v3/test_store/test_local.py index 59cae22de3..5f1dde3fcc 100644 --- a/tests/v3/test_store/test_local.py +++ b/tests/v3/test_store/test_local.py @@ -35,6 +35,3 @@ def test_store_supports_partial_writes(self, store: LocalStore) -> None: def test_store_supports_listing(self, store: LocalStore) -> None: assert store.supports_listing - - def test_list_prefix(self, store: LocalStore) -> None: - assert True diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index afa991209f..e2c3070198 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -1,13 +1,21 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Generator + + import botocore.client + import os -from collections.abc import Generator -import botocore.client import fsspec import pytest +from botocore.session import Session from upath import UPath from zarr.core.buffer import Buffer, cpu, default_buffer_prototype -from zarr.core.sync import sync +from zarr.core.sync import _collect_aiterator, sync from zarr.store import RemoteStore from zarr.testing.store import StoreTests @@ -40,8 +48,6 @@ def s3_base() -> Generator[None, None, None]: def get_boto3_client() -> botocore.client.BaseClient: - from botocore.session import Session - # NB: we use the sync botocore client for setup session = Session() return session.create_client("s3", endpoint_url=endpoint_url) @@ -87,7 +93,7 @@ async def test_basic() -> None: store = await RemoteStore.open( f"s3://{test_bucket_name}", mode="w", endpoint_url=endpoint_url, anon=False ) - assert not await alist(store.list()) + assert await _collect_aiterator(store.list()) == () assert not await store.exists("foo") data = b"hello" await store.set("foo", cpu.Buffer.from_bytes(data)) @@ -104,7 +110,7 @@ class TestRemoteStoreS3(StoreTests[RemoteStore, cpu.Buffer]): buffer_cls = cpu.Buffer @pytest.fixture(scope="function", params=("use_upath", "use_str")) - def store_kwargs(self, request) -> dict[str, str | bool]: + def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, str | bool | UPath]: # type: ignore url = f"s3://{test_bucket_name}" anon = False mode = "r+" @@ -116,8 +122,8 @@ def store_kwargs(self, request) -> dict[str, str | bool]: raise AssertionError @pytest.fixture(scope="function") - def store(self, store_kwargs: dict[str, str | bool]) -> RemoteStore: - url = store_kwargs["url"] + async def store(self, store_kwargs: dict[str, str | bool | UPath]) -> RemoteStore: + url: str | UPath = store_kwargs["url"] mode = store_kwargs["mode"] if isinstance(url, UPath): out = self.store_cls(url=url, mode=mode) From 54060d3ef84a269ed7b7b561bfd62341b359045a Mon Sep 17 00:00:00 2001 From: Lindsey Nield Date: Thu, 19 Sep 2024 11:38:08 -0600 Subject: [PATCH 0164/1043] [v3] Implement Group methods for empty, full, ones, and zeros (#2210) * fill in stubs for Group.{empty,zeros,ones,full,empty_like, zeros_like,onest_like,full_like} * add shape to function signature * change type in function signature and add unit tests * precommit * small fixes * add shape check to tests * update function signatures * cast path to a str * update store path --------- Co-authored-by: Joe Hamman --- src/zarr/api/asynchronous.py | 4 +- src/zarr/core/group.py | 91 +++++++++++++++++++++++------------- tests/v3/test_group.py | 67 ++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 34 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 8a1b0c5f36..38576d73de 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -575,7 +575,7 @@ async def create( chunks: ChunkCoords | None = None, # TODO: v2 allowed chunks=True dtype: npt.DTypeLike | None = None, compressor: dict[str, JSON] | None = None, # TODO: default and type change - fill_value: Any = 0, # TODO: need type + fill_value: Any | None = 0, # TODO: need type order: MemoryOrder | None = None, # TODO: default change store: str | StoreLike | None = None, synchronizer: Any | None = None, @@ -827,7 +827,7 @@ async def full_like(a: ArrayLike, **kwargs: Any) -> AsyncArray: """ like_kwargs = _like_args(a, kwargs) if isinstance(a, AsyncArray): - kwargs.setdefault("fill_value", a.metadata.fill_value) + like_kwargs.setdefault("fill_value", a.metadata.fill_value) return await full(**like_kwargs) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 40815b96c8..a4e1e252ea 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -10,6 +10,7 @@ import numpy.typing as npt from typing_extensions import deprecated +import zarr.api.asynchronous as async_api from zarr.abc.metadata import Metadata from zarr.abc.store import set_or_delete from zarr.core.array import Array, AsyncArray @@ -704,29 +705,41 @@ async def arrays(self) -> AsyncGenerator[AsyncArray, None]: async def tree(self, expand: bool = False, level: int | None = None) -> Any: raise NotImplementedError - async def empty(self, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> AsyncArray: + return await async_api.empty(shape=shape, store=self.store_path, path=name, **kwargs) - async def zeros(self, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def zeros(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> AsyncArray: + return await async_api.zeros(shape=shape, store=self.store_path, path=name, **kwargs) - async def ones(self, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def ones(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> AsyncArray: + return await async_api.ones(shape=shape, store=self.store_path, path=name, **kwargs) - async def full(self, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def full( + self, *, name: str, shape: ChunkCoords, fill_value: Any | None, **kwargs: Any + ) -> AsyncArray: + return await async_api.full( + shape=shape, fill_value=fill_value, store=self.store_path, path=name, **kwargs + ) - async def empty_like(self, prototype: AsyncArray, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def empty_like( + self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray: + return await async_api.empty_like(a=prototype, store=self.store_path, path=name, **kwargs) - async def zeros_like(self, prototype: AsyncArray, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def zeros_like( + self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray: + return await async_api.zeros_like(a=prototype, store=self.store_path, path=name, **kwargs) - async def ones_like(self, prototype: AsyncArray, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def ones_like( + self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray: + return await async_api.ones_like(a=prototype, store=self.store_path, path=name, **kwargs) - async def full_like(self, prototype: AsyncArray, **kwargs: Any) -> AsyncArray: - raise NotImplementedError + async def full_like( + self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any + ) -> AsyncArray: + return await async_api.full_like(a=prototype, store=self.store_path, path=name, **kwargs) async def move(self, source: str, dest: str) -> None: raise NotImplementedError @@ -1058,29 +1071,43 @@ def require_array(self, name: str, **kwargs: Any) -> Array: """ return Array(self._sync(self._async_group.require_array(name, **kwargs))) - def empty(self, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.empty(**kwargs))) + def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: + return Array(self._sync(self._async_group.empty(name=name, shape=shape, **kwargs))) - def zeros(self, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.zeros(**kwargs))) + def zeros(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: + return Array(self._sync(self._async_group.zeros(name=name, shape=shape, **kwargs))) - def ones(self, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.ones(**kwargs))) + def ones(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: + return Array(self._sync(self._async_group.ones(name=name, shape=shape, **kwargs))) - def full(self, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.full(**kwargs))) + def full( + self, *, name: str, shape: ChunkCoords, fill_value: Any | None, **kwargs: Any + ) -> Array: + return Array( + self._sync( + self._async_group.full(name=name, shape=shape, fill_value=fill_value, **kwargs) + ) + ) - def empty_like(self, prototype: AsyncArray, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.empty_like(prototype, **kwargs))) + def empty_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array: + return Array( + self._sync(self._async_group.empty_like(name=name, prototype=prototype, **kwargs)) + ) - def zeros_like(self, prototype: AsyncArray, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.zeros_like(prototype, **kwargs))) + def zeros_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array: + return Array( + self._sync(self._async_group.zeros_like(name=name, prototype=prototype, **kwargs)) + ) - def ones_like(self, prototype: AsyncArray, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.ones_like(prototype, **kwargs))) + def ones_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array: + return Array( + self._sync(self._async_group.ones_like(name=name, prototype=prototype, **kwargs)) + ) - def full_like(self, prototype: AsyncArray, **kwargs: Any) -> Array: - return Array(self._sync(self._async_group.full_like(prototype, **kwargs))) + def full_like(self, *, name: str, prototype: async_api.ArrayLike, **kwargs: Any) -> Array: + return Array( + self._sync(self._async_group.full_like(name=name, prototype=prototype, **kwargs)) + ) def move(self, source: str, dest: str) -> None: return self._sync(self._async_group.move(source, dest)) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index d5fb9e7b5a..4bb23fddaa 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -390,6 +390,73 @@ def test_group_create_array( assert np.array_equal(array[:], data) +def test_group_array_creation( + store: Store, + zarr_format: ZarrFormat, +): + group = Group.create(store, zarr_format=zarr_format) + shape = (10, 10) + empty_array = group.empty(name="empty", shape=shape) + assert isinstance(empty_array, Array) + assert empty_array.fill_value == 0 + assert empty_array.shape == shape + assert empty_array.store_path.store == store + + empty_like_array = group.empty_like(name="empty_like", prototype=empty_array) + assert isinstance(empty_like_array, Array) + assert empty_like_array.fill_value == 0 + assert empty_like_array.shape == shape + assert empty_like_array.store_path.store == store + + empty_array_bool = group.empty(name="empty_bool", shape=shape, dtype=np.dtype("bool")) + assert isinstance(empty_array_bool, Array) + assert not empty_array_bool.fill_value + assert empty_array_bool.shape == shape + assert empty_array_bool.store_path.store == store + + empty_like_array_bool = group.empty_like(name="empty_like_bool", prototype=empty_array_bool) + assert isinstance(empty_like_array_bool, Array) + assert not empty_like_array_bool.fill_value + assert empty_like_array_bool.shape == shape + assert empty_like_array_bool.store_path.store == store + + zeros_array = group.zeros(name="zeros", shape=shape) + assert isinstance(zeros_array, Array) + assert zeros_array.fill_value == 0 + assert zeros_array.shape == shape + assert zeros_array.store_path.store == store + + zeros_like_array = group.zeros_like(name="zeros_like", prototype=zeros_array) + assert isinstance(zeros_like_array, Array) + assert zeros_like_array.fill_value == 0 + assert zeros_like_array.shape == shape + assert zeros_like_array.store_path.store == store + + ones_array = group.ones(name="ones", shape=shape) + assert isinstance(ones_array, Array) + assert ones_array.fill_value == 1 + assert ones_array.shape == shape + assert ones_array.store_path.store == store + + ones_like_array = group.ones_like(name="ones_like", prototype=ones_array) + assert isinstance(ones_like_array, Array) + assert ones_like_array.fill_value == 1 + assert ones_like_array.shape == shape + assert ones_like_array.store_path.store == store + + full_array = group.full(name="full", shape=shape, fill_value=42) + assert isinstance(full_array, Array) + assert full_array.fill_value == 42 + assert full_array.shape == shape + assert full_array.store_path.store == store + + full_like_array = group.full_like(name="full_like", prototype=full_array, fill_value=43) + assert isinstance(full_like_array, Array) + assert full_like_array.fill_value == 43 + assert full_like_array.shape == shape + assert full_like_array.store_path.store == store + + @pytest.mark.parametrize("store", ("local", "memory", "zip"), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) @pytest.mark.parametrize("exists_ok", [True, False]) From f8943351b0424e06856271275a9c3c93140775b2 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 19 Sep 2024 12:40:20 -0500 Subject: [PATCH 0165/1043] Make MemoryStore serialiazable (#2204) --- src/zarr/store/memory.py | 8 +------- tests/v3/test_store/test_memory.py | 22 ---------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/src/zarr/store/memory.py b/src/zarr/store/memory.py index e304419768..7baa6aee26 100644 --- a/src/zarr/store/memory.py +++ b/src/zarr/store/memory.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import AsyncGenerator, MutableMapping -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from zarr.abc.store import Store from zarr.core.buffer import Buffer, gpu @@ -55,12 +55,6 @@ def __eq__(self, other: object) -> bool: and self.mode == other.mode ) - def __setstate__(self, state: Any) -> None: - raise NotImplementedError(f"{type(self)} cannot be pickled") - - def __getstate__(self) -> None: - raise NotImplementedError(f"{type(self)} cannot be pickled") - async def get( self, key: str, diff --git a/tests/v3/test_store/test_memory.py b/tests/v3/test_store/test_memory.py index 470383dfb6..2498cdc24a 100644 --- a/tests/v3/test_store/test_memory.py +++ b/tests/v3/test_store/test_memory.py @@ -1,7 +1,5 @@ from __future__ import annotations -import pickle - import pytest from zarr.core.buffer import Buffer, cpu, gpu @@ -48,16 +46,6 @@ def test_store_supports_partial_writes(self, store: MemoryStore) -> None: def test_list_prefix(self, store: MemoryStore) -> None: assert True - def test_serizalizable_store(self, store: MemoryStore) -> None: - with pytest.raises(NotImplementedError): - store.__getstate__() - - with pytest.raises(NotImplementedError): - store.__setstate__({}) - - with pytest.raises(NotImplementedError): - pickle.dumps(store) - @gpu_test class TestGpuMemoryStore(StoreTests[GpuMemoryStore, gpu.Buffer]): @@ -92,13 +80,3 @@ def test_store_supports_partial_writes(self, store: GpuMemoryStore) -> None: def test_list_prefix(self, store: GpuMemoryStore) -> None: assert True - - def test_serizalizable_store(self, store: MemoryStore) -> None: - with pytest.raises(NotImplementedError): - store.__getstate__() - - with pytest.raises(NotImplementedError): - store.__setstate__({}) - - with pytest.raises(NotImplementedError): - pickle.dumps(store) From f538662167480fe054c7919d0d51b8e16e6aad70 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 19 Sep 2024 12:44:14 -0500 Subject: [PATCH 0166/1043] Default to RemoteStore for fsspec URIs (#2198) * Default to RemoteStore for fsspec URIs * fixup * fixup * wip * fixup * Added check for invalid. * fixup * fixup --------- Co-authored-by: Joe Hamman --- src/zarr/api/asynchronous.py | 61 ++++++++++++--- src/zarr/api/synchronous.py | 2 + src/zarr/store/common.py | 57 ++++++++++++-- src/zarr/store/remote.py | 106 +++++++++++++------------- tests/v3/test_store/test_core.py | 33 ++++++++- tests/v3/test_store/test_remote.py | 115 ++++++++++++++++++----------- 6 files changed, 260 insertions(+), 114 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 38576d73de..58278172f7 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -194,6 +194,7 @@ async def open( zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, + storage_options: dict[str, Any] | None = None, **kwargs: Any, # TODO: type kwargs as valid args to open_array ) -> AsyncArray | AsyncGroup: """Convenience function to open a group or array using file-mode-like semantics. @@ -211,6 +212,9 @@ async def open( The zarr format to use when saving. path : str or None, optional The path within the store to open. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. **kwargs Additional parameters are passed through to :func:`zarr.creation.open_array` or :func:`zarr.hierarchy.open_group`. @@ -221,7 +225,7 @@ async def open( Return type depends on what exists in the given store. """ zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) - store_path = await make_store_path(store, mode=mode) + store_path = await make_store_path(store, mode=mode, storage_options=storage_options) if path is not None: store_path = store_path / path @@ -276,6 +280,7 @@ async def save_array( zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, + storage_options: dict[str, Any] | None = None, **kwargs: Any, # TODO: type kwargs as valid args to create ) -> None: """Convenience function to save a NumPy array to the local file system, following a @@ -291,6 +296,9 @@ async def save_array( The zarr format to use when saving. path : str or None, optional The path within the store where the array will be saved. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. kwargs Passed through to :func:`create`, e.g., compressor. """ @@ -299,7 +307,7 @@ async def save_array( or _default_zarr_version() ) - store_path = await make_store_path(store, mode="w") + store_path = await make_store_path(store, mode="w", storage_options=storage_options) if path is not None: store_path = store_path / path new = await AsyncArray.create( @@ -319,6 +327,7 @@ async def save_group( zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, + storage_options: dict[str, Any] | None = None, **kwargs: NDArrayLike, ) -> None: """Convenience function to save several NumPy arrays to the local file system, following a @@ -334,11 +343,17 @@ async def save_group( The zarr format to use when saving. path : str or None, optional Path within the store where the group will be saved. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. kwargs NumPy arrays with data to save. """ zarr_format = ( - _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) + _handle_zarr_version_or_format( + zarr_version=zarr_version, + zarr_format=zarr_format, + ) or _default_zarr_version() ) @@ -346,10 +361,22 @@ async def save_group( raise ValueError("at least one array must be provided") aws = [] for i, arr in enumerate(args): - aws.append(save_array(store, arr, zarr_format=zarr_format, path=f"{path}/arr_{i}")) + aws.append( + save_array( + store, + arr, + zarr_format=zarr_format, + path=f"{path}/arr_{i}", + storage_options=storage_options, + ) + ) for k, arr in kwargs.items(): _path = f"{path}/{k}" if path is not None else k - aws.append(save_array(store, arr, zarr_format=zarr_format, path=_path)) + aws.append( + save_array( + store, arr, zarr_format=zarr_format, path=_path, storage_options=storage_options + ) + ) await asyncio.gather(*aws) @@ -418,6 +445,7 @@ async def group( zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # not used attributes: dict[str, JSON] | None = None, + storage_options: dict[str, Any] | None = None, ) -> AsyncGroup: """Create a group. @@ -444,6 +472,9 @@ async def group( to users. Use `numpy.empty(())` by default. zarr_format : {2, 3, None}, optional The zarr format to use when saving. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. Returns ------- @@ -453,7 +484,7 @@ async def group( zarr_format = _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) - store_path = await make_store_path(store) + store_path = await make_store_path(store, storage_options=storage_options) if path is not None: store_path = store_path / path @@ -488,7 +519,7 @@ async def open_group( synchronizer: Any = None, # not used path: str | None = None, chunk_store: StoreLike | None = None, # not used - storage_options: dict[str, Any] | None = None, # not used + storage_options: dict[str, Any] | None = None, zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, meta_array: Any | None = None, # not used @@ -548,10 +579,8 @@ async def open_group( warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2) if chunk_store is not None: warnings.warn("chunk_store is not yet implemented", RuntimeWarning, stacklevel=2) - if storage_options is not None: - warnings.warn("storage_options is not yet implemented", RuntimeWarning, stacklevel=2) - store_path = await make_store_path(store, mode=mode) + store_path = await make_store_path(store, mode=mode, storage_options=storage_options) if path is not None: store_path = store_path / path @@ -603,6 +632,7 @@ async def create( ) = None, codecs: Iterable[Codec | dict[str, JSON]] | None = None, dimension_names: Iterable[str] | None = None, + storage_options: dict[str, Any] | None = None, **kwargs: Any, ) -> AsyncArray: """Create an array. @@ -674,6 +704,9 @@ async def create( to users. Use `numpy.empty(())` by default. .. versionadded:: 2.13 + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. Returns ------- @@ -725,7 +758,7 @@ async def create( warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2) mode = kwargs.pop("mode", cast(AccessModeLiteral, "r" if read_only else "w")) - store_path = await make_store_path(store, mode=mode) + store_path = await make_store_path(store, mode=mode, storage_options=storage_options) if path is not None: store_path = store_path / path @@ -875,6 +908,7 @@ async def open_array( zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: PathLike | None = None, + storage_options: dict[str, Any] | None = None, **kwargs: Any, # TODO: type kwargs as valid args to save ) -> AsyncArray: """Open an array using file-mode-like semantics. @@ -887,6 +921,9 @@ async def open_array( The zarr format to use when saving. path : string, optional Path in store to array. + storage_options : dict + If using an fsspec URL to create the store, these will be passed to + the backend implementation. Ignored otherwise. **kwargs Any keyword arguments to pass to the array constructor. @@ -896,7 +933,7 @@ async def open_array( The opened array. """ - store_path = await make_store_path(store) + store_path = await make_store_path(store, storage_options=storage_options) if path is not None: store_path = store_path / path diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 93a33b8d3f..129f901934 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -134,6 +134,7 @@ def save_group( zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, path: str | None = None, + storage_options: dict[str, Any] | None = None, **kwargs: NDArrayLike, ) -> None: return sync( @@ -143,6 +144,7 @@ def save_group( zarr_version=zarr_version, zarr_format=zarr_format, path=path, + storage_options=storage_options, **kwargs, ) ) diff --git a/src/zarr/store/common.py b/src/zarr/store/common.py index 8028c9af3d..196479dd67 100644 --- a/src/zarr/store/common.py +++ b/src/zarr/store/common.py @@ -11,6 +11,8 @@ from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore +# from zarr.store.remote import RemoteStore + if TYPE_CHECKING: from zarr.core.buffer import BufferPrototype from zarr.core.common import AccessModeLiteral @@ -75,30 +77,69 @@ def __eq__(self, other: Any) -> bool: async def make_store_path( - store_like: StoreLike | None, *, mode: AccessModeLiteral | None = None + store_like: StoreLike | None, + *, + mode: AccessModeLiteral | None = None, + storage_options: dict[str, Any] | None = None, ) -> StorePath: + from zarr.store.remote import RemoteStore # circular import + + used_storage_options = False + if isinstance(store_like, StorePath): if mode is not None: assert AccessMode.from_literal(mode) == store_like.store.mode - return store_like + result = store_like elif isinstance(store_like, Store): if mode is not None: assert AccessMode.from_literal(mode) == store_like.mode await store_like._ensure_open() - return StorePath(store_like) + result = StorePath(store_like) elif store_like is None: if mode is None: mode = "w" # exception to the default mode = 'r' - return StorePath(await MemoryStore.open(mode=mode)) + result = StorePath(await MemoryStore.open(mode=mode)) elif isinstance(store_like, Path): - return StorePath(await LocalStore.open(root=store_like, mode=mode or "r")) + result = StorePath(await LocalStore.open(root=store_like, mode=mode or "r")) elif isinstance(store_like, str): - return StorePath(await LocalStore.open(root=Path(store_like), mode=mode or "r")) + storage_options = storage_options or {} + + if _is_fsspec_uri(store_like): + used_storage_options = True + result = StorePath( + RemoteStore.from_url(store_like, storage_options=storage_options, mode=mode or "r") + ) + else: + result = StorePath(await LocalStore.open(root=Path(store_like), mode=mode or "r")) elif isinstance(store_like, dict): # We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. # By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. - return StorePath(await MemoryStore.open(store_dict=store_like, mode=mode)) - raise TypeError + result = StorePath(await MemoryStore.open(store_dict=store_like, mode=mode)) + else: + msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] + raise TypeError(msg) + + if storage_options and not used_storage_options: + msg = "'storage_options' was provided but unused. 'storage_options' is only used for fsspec filesystem stores." + raise TypeError(msg) + + return result + + +def _is_fsspec_uri(uri: str) -> bool: + """ + Check if a URI looks like a non-local fsspec URI. + + Examples + -------- + >>> _is_fsspec_uri("s3://bucket") + True + >>> _is_fsspec_uri("my-directory") + False + >>> _is_fsspec_uri("local://my-directory") + False + """ + return "://" in uri or "::" in uri and "local://" not in uri async def ensure_no_existing_node(store_path: StorePath, zarr_format: ZarrFormat) -> None: diff --git a/src/zarr/store/remote.py b/src/zarr/store/remote.py index 084ef986b1..ecb46a31d3 100644 --- a/src/zarr/store/remote.py +++ b/src/zarr/store/remote.py @@ -11,12 +11,18 @@ from collections.abc import AsyncGenerator from fsspec.asyn import AsyncFileSystem - from upath import UPath from zarr.core.buffer import Buffer, BufferPrototype from zarr.core.common import AccessModeLiteral, BytesLike +ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = ( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, +) + + class RemoteStore(Store): # based on FSSpec supports_writes: bool = True @@ -24,21 +30,15 @@ class RemoteStore(Store): supports_partial_writes: bool = False supports_listing: bool = True - _fs: AsyncFileSystem - _url: str - path: str + fs: AsyncFileSystem allowed_exceptions: tuple[type[Exception], ...] def __init__( self, - url: UPath | str, + fs: AsyncFileSystem, mode: AccessModeLiteral = "r", - allowed_exceptions: tuple[type[Exception], ...] = ( - FileNotFoundError, - IsADirectoryError, - NotADirectoryError, - ), - **storage_options: Any, + path: str = "/", + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, ): """ Parameters @@ -51,54 +51,58 @@ def __init__( this must not be used. """ super().__init__(mode=mode) - self._storage_options = storage_options - if isinstance(url, str): - self._url = url.rstrip("/") - self._fs, _path = fsspec.url_to_fs(url, **storage_options) - self.path = _path.rstrip("/") - elif hasattr(url, "protocol") and hasattr(url, "fs"): - # is UPath-like - but without importing - if storage_options: - raise ValueError( - "If constructed with a UPath object, no additional " - "storage_options are allowed" - ) - # n.b. UPath returns the url and path attributes with a trailing /, at least for s3 - # that trailing / must be removed to compose with the store interface - self._url = str(url).rstrip("/") - self.path = url.path.rstrip("/") - self._fs = url.fs - else: - raise ValueError(f"URL not understood, {url}") + self.fs = fs + self.path = path self.allowed_exceptions = allowed_exceptions - # test instantiate file system - if not self._fs.async_impl: - raise TypeError("FileSystem needs to support async operations") + + if not self.fs.async_impl: + raise TypeError("Filesystem needs to support async operations.") + + @classmethod + def from_upath( + cls, + upath: Any, + mode: AccessModeLiteral = "r", + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> RemoteStore: + return cls( + fs=upath.fs, + path=upath.path.rstrip("/"), + mode=mode, + allowed_exceptions=allowed_exceptions, + ) + + @classmethod + def from_url( + cls, + url: str, + storage_options: dict[str, Any] | None = None, + mode: AccessModeLiteral = "r", + allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS, + ) -> RemoteStore: + fs, path = fsspec.url_to_fs(url, **storage_options) + return cls(fs=fs, path=path, mode=mode, allowed_exceptions=allowed_exceptions) async def clear(self) -> None: try: - for subpath in await self._fs._find(self.path, withdirs=True): + for subpath in await self.fs._find(self.path, withdirs=True): if subpath != self.path: - await self._fs._rm(subpath, recursive=True) + await self.fs._rm(subpath, recursive=True) except FileNotFoundError: pass async def empty(self) -> bool: - return not await self._fs._find(self.path, withdirs=True) - - def __str__(self) -> str: - return f"{self._url}" + return not await self.fs._find(self.path, withdirs=True) def __repr__(self) -> str: - return f"" + return f"" def __eq__(self, other: object) -> bool: return ( isinstance(other, type(self)) and self.path == other.path and self.mode == other.mode - and self._url == other._url - # and self._storage_options == other._storage_options # FIXME: this isn't working for some reason + and self.fs == other.fs ) async def get( @@ -123,9 +127,9 @@ async def get( end = None value = prototype.buffer.from_bytes( await ( - self._fs._cat_file(path, start=byte_range[0], end=end) + self.fs._cat_file(path, start=byte_range[0], end=end) if byte_range - else self._fs._cat_file(path) + else self.fs._cat_file(path) ) ) @@ -152,13 +156,13 @@ async def set( # write data if byte_range: raise NotImplementedError - await self._fs._pipe_file(path, value.to_bytes()) + await self.fs._pipe_file(path, value.to_bytes()) async def delete(self, key: str) -> None: self._check_writable() path = _dereference_path(self.path, key) try: - await self._fs._rm(path) + await self.fs._rm(path) except FileNotFoundError: pass except self.allowed_exceptions: @@ -166,7 +170,7 @@ async def delete(self, key: str) -> None: async def exists(self, key: str) -> bool: path = _dereference_path(self.path, key) - exists: bool = await self._fs._exists(path) + exists: bool = await self.fs._exists(path) return exists async def get_partial_values( @@ -189,7 +193,7 @@ async def get_partial_values( else: return [] # TODO: expectations for exceptions or missing keys? - res = await self._fs._cat_ranges(list(paths), starts, stops, on_error="return") + res = await self.fs._cat_ranges(list(paths), starts, stops, on_error="return") # the following is an s3-specific condition we probably don't want to leak res = [b"" if (isinstance(r, OSError) and "not satisfiable" in str(r)) else r for r in res] for r in res: @@ -202,14 +206,14 @@ async def set_partial_values(self, key_start_values: list[tuple[str, int, BytesL raise NotImplementedError async def list(self) -> AsyncGenerator[str, None]: - allfiles = await self._fs._find(self.path, detail=False, withdirs=False) + allfiles = await self.fs._find(self.path, detail=False, withdirs=False) for onefile in (a.replace(self.path + "/", "") for a in allfiles): yield onefile async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: prefix = f"{self.path}/{prefix.rstrip('/')}" try: - allfiles = await self._fs._ls(prefix, detail=False) + allfiles = await self.fs._ls(prefix, detail=False) except FileNotFoundError: return for onefile in (a.replace(prefix + "/", "") for a in allfiles): @@ -230,5 +234,5 @@ async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: """ find_str = "/".join([self.path, prefix]) - for onefile in await self._fs._find(find_str, detail=False, maxdepth=None, withdirs=False): + for onefile in await self.fs._find(find_str, detail=False, maxdepth=None, withdirs=False): yield onefile.removeprefix(find_str) diff --git a/tests/v3/test_store/test_core.py b/tests/v3/test_store/test_core.py index c65d91f9d0..f401491127 100644 --- a/tests/v3/test_store/test_core.py +++ b/tests/v3/test_store/test_core.py @@ -1,10 +1,12 @@ +import tempfile from pathlib import Path import pytest -from zarr.store.common import make_store_path +from zarr.store.common import StoreLike, StorePath, make_store_path from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore +from zarr.store.remote import RemoteStore async def test_make_store_path(tmpdir: str) -> None: @@ -34,3 +36,32 @@ async def test_make_store_path(tmpdir: str) -> None: with pytest.raises(TypeError): await make_store_path(1) # type: ignore[arg-type] + + +async def test_make_store_path_fsspec(monkeypatch) -> None: + import fsspec.implementations.memory + + monkeypatch.setattr(fsspec.implementations.memory.MemoryFileSystem, "async_impl", True) + store_path = await make_store_path("memory://") + assert isinstance(store_path.store, RemoteStore) + + +@pytest.mark.parametrize( + "store_like", + [ + None, + str(tempfile.TemporaryDirectory()), + Path(tempfile.TemporaryDirectory().name), + StorePath(store=MemoryStore(store_dict={}, mode="w"), path="/"), + MemoryStore(store_dict={}, mode="w"), + {}, + ], +) +async def test_make_store_path_storage_options_raises(store_like: StoreLike) -> None: + with pytest.raises(TypeError, match="storage_options"): + await make_store_path(store_like, storage_options={"foo": "bar"}, mode="w") + + +async def test_unsupported() -> None: + with pytest.raises(TypeError, match="Unsupported type for store_like: 'int'"): + await make_store_path(1) # type: ignore[arg-type] diff --git a/tests/v3/test_store/test_remote.py b/tests/v3/test_store/test_remote.py index e2c3070198..495a5e5c4f 100644 --- a/tests/v3/test_store/test_remote.py +++ b/tests/v3/test_store/test_remote.py @@ -1,24 +1,27 @@ from __future__ import annotations -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Generator - - import botocore.client - +import json import os +from collections.abc import Generator +from typing import TYPE_CHECKING import fsspec import pytest from botocore.session import Session from upath import UPath +import zarr.api.asynchronous from zarr.core.buffer import Buffer, cpu, default_buffer_prototype from zarr.core.sync import _collect_aiterator, sync from zarr.store import RemoteStore from zarr.testing.store import StoreTests +if TYPE_CHECKING: + from collections.abc import Generator + + import botocore.client + + s3fs = pytest.importorskip("s3fs") requests = pytest.importorskip("requests") moto_server = pytest.importorskip("moto.moto_server.threaded_moto_server") @@ -90,8 +93,10 @@ async def alist(it): async def test_basic() -> None: - store = await RemoteStore.open( - f"s3://{test_bucket_name}", mode="w", endpoint_url=endpoint_url, anon=False + store = RemoteStore.from_url( + f"s3://{test_bucket_name}", + mode="w", + storage_options=dict(endpoint_url=endpoint_url, anon=False), ) assert await _collect_aiterator(store.list()) == () assert not await store.exists("foo") @@ -109,51 +114,33 @@ class TestRemoteStoreS3(StoreTests[RemoteStore, cpu.Buffer]): store_cls = RemoteStore buffer_cls = cpu.Buffer - @pytest.fixture(scope="function", params=("use_upath", "use_str")) - def store_kwargs(self, request: pytest.FixtureRequest) -> dict[str, str | bool | UPath]: # type: ignore - url = f"s3://{test_bucket_name}" - anon = False - mode = "r+" - if request.param == "use_upath": - return {"url": UPath(url, endpoint_url=endpoint_url, anon=anon), "mode": mode} - elif request.param == "use_str": - return {"url": url, "mode": mode, "anon": anon, "endpoint_url": endpoint_url} - - raise AssertionError + @pytest.fixture(scope="function") + def store_kwargs(self, request) -> dict[str, str | bool]: + fs, path = fsspec.url_to_fs( + f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False + ) + return {"fs": fs, "path": path, "mode": "r+"} @pytest.fixture(scope="function") - async def store(self, store_kwargs: dict[str, str | bool | UPath]) -> RemoteStore: - url: str | UPath = store_kwargs["url"] - mode = store_kwargs["mode"] - if isinstance(url, UPath): - out = self.store_cls(url=url, mode=mode) - else: - endpoint_url = store_kwargs["endpoint_url"] - out = self.store_cls(url=url, asynchronous=True, mode=mode, endpoint_url=endpoint_url) - return out + def store(self, store_kwargs: dict[str, str | bool]) -> RemoteStore: + return self.store_cls(**store_kwargs) def get(self, store: RemoteStore, key: str) -> Buffer: # make a new, synchronous instance of the filesystem because this test is run in sync code - fs, _ = fsspec.url_to_fs( - url=store._url, - asynchronous=False, - anon=store._fs.anon, - endpoint_url=store._fs.endpoint_url, + new_fs = fsspec.filesystem( + "s3", endpoint_url=store.fs.endpoint_url, anon=store.fs.anon, asynchronous=False ) - return self.buffer_cls.from_bytes(fs.cat(f"{store.path}/{key}")) + return self.buffer_cls.from_bytes(new_fs.cat(f"{store.path}/{key}")) def set(self, store: RemoteStore, key: str, value: Buffer) -> None: # make a new, synchronous instance of the filesystem because this test is run in sync code - fs, _ = fsspec.url_to_fs( - url=store._url, - asynchronous=False, - anon=store._fs.anon, - endpoint_url=store._fs.endpoint_url, + new_fs = fsspec.filesystem( + "s3", endpoint_url=store.fs.endpoint_url, anon=store.fs.anon, asynchronous=False ) - fs.write_bytes(f"{store.path}/{key}", value.to_bytes()) + new_fs.write_bytes(f"{store.path}/{key}", value.to_bytes()) def test_store_repr(self, store: RemoteStore) -> None: - assert str(store) == f"s3://{test_bucket_name}" + assert str(store) == "" def test_store_supports_writes(self, store: RemoteStore) -> None: assert True @@ -164,3 +151,47 @@ def test_store_supports_partial_writes(self, store: RemoteStore) -> None: def test_store_supports_listing(self, store: RemoteStore) -> None: assert True + + async def test_remote_store_from_uri( + self, store: RemoteStore, store_kwargs: dict[str, str | bool] + ): + storage_options = { + "endpoint_url": endpoint_url, + "anon": False, + } + + meta = {"attributes": {"key": "value"}, "zarr_format": 3, "node_type": "group"} + + await store.set( + "zarr.json", + self.buffer_cls.from_bytes(json.dumps(meta).encode()), + ) + group = await zarr.api.asynchronous.open_group( + store=f"s3://{test_bucket_name}", storage_options=storage_options + ) + assert dict(group.attrs) == {"key": "value"} + + meta["attributes"]["key"] = "value-2" + await store.set( + "directory-2/zarr.json", + self.buffer_cls.from_bytes(json.dumps(meta).encode()), + ) + group = await zarr.api.asynchronous.open_group( + store=f"s3://{test_bucket_name}/directory-2", storage_options=storage_options + ) + assert dict(group.attrs) == {"key": "value-2"} + + meta["attributes"]["key"] = "value-3" + await store.set( + "directory-3/zarr.json", + self.buffer_cls.from_bytes(json.dumps(meta).encode()), + ) + group = await zarr.api.asynchronous.open_group( + store=f"s3://{test_bucket_name}", path="directory-3", storage_options=storage_options + ) + assert dict(group.attrs) == {"key": "value-3"} + + def test_from_upath(self) -> None: + path = UPath(f"s3://{test_bucket_name}", endpoint_url=endpoint_url, anon=False) + result = RemoteStore.from_upath(path) + assert result.fs.endpoint_url == endpoint_url From 32540b4fa98e6d9ca72b29134cf015333acc7c97 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Thu, 19 Sep 2024 23:01:09 +0100 Subject: [PATCH 0167/1043] Typing fixes to test_indexing (#2193) --- tests/v3/test_indexing.py | 215 +++++++++++++++++++++++++------------- 1 file changed, 142 insertions(+), 73 deletions(-) diff --git a/tests/v3/test_indexing.py b/tests/v3/test_indexing.py index efb11f36a1..8b509f93d1 100644 --- a/tests/v3/test_indexing.py +++ b/tests/v3/test_indexing.py @@ -12,6 +12,10 @@ import zarr from zarr.core.buffer import BufferPrototype, default_buffer_prototype from zarr.core.indexing import ( + BasicSelection, + CoordinateSelection, + OrthogonalSelection, + Selection, make_slice_selection, normalize_integer_selection, oindex, @@ -23,13 +27,15 @@ from zarr.store.memory import MemoryStore if TYPE_CHECKING: - from collections.abc import Iterator + from collections.abc import AsyncGenerator + from zarr.core.array import Array + from zarr.core.buffer.core import Buffer from zarr.core.common import ChunkCoords @pytest.fixture -async def store() -> Iterator[StorePath]: +async def store() -> AsyncGenerator[StorePath]: yield StorePath(await MemoryStore.open(mode="w")) @@ -50,18 +56,25 @@ def zarr_array_from_numpy_array( class CountingDict(MemoryStore): + counter: Counter[tuple[str, str]] + @classmethod async def open(cls) -> CountingDict: store = await super().open(mode="w") store.counter = Counter() return store - async def get(self, key, prototype: BufferPrototype, byte_range=None): + async def get( + self, + key: str, + prototype: BufferPrototype, + byte_range: tuple[int | None, int | None] | None = None, + ) -> Buffer | None: key_suffix = "/".join(key.split("/")[1:]) self.counter["__getitem__", key_suffix] += 1 return await super().get(key, prototype, byte_range) - async def set(self, key, value, byte_range=None): + async def set(self, key: str, value: Buffer, byte_range: tuple[int, int] | None = None) -> None: key_suffix = "/".join(key.split("/")[1:]) self.counter["__setitem__", key_suffix] += 1 return await super().set(key, value, byte_range) @@ -167,7 +180,7 @@ def test_get_basic_selection_0d(store: StorePath, use_out: bool, value: Any, dty # assert_array_equal(a[["foo", "bar"]], c) -basic_selections_1d = [ +basic_selections_1d: list[BasicSelection] = [ # single value 42, -1, @@ -241,7 +254,9 @@ def test_get_basic_selection_0d(store: StorePath, use_out: bool, value: Any, dty ] -def _test_get_basic_selection(a, z, selection) -> None: +def _test_get_basic_selection( + a: npt.NDArray[Any] | Array, z: Array, selection: BasicSelection +) -> None: expect = a[selection] actual = z.get_basic_selection(selection) assert_array_equal(expect, actual) @@ -265,17 +280,17 @@ def test_get_basic_selection_1d(store: StorePath) -> None: for selection in basic_selections_1d: _test_get_basic_selection(a, z, selection) - for selection in basic_selections_1d_bad: + for selection_bad in basic_selections_1d_bad: with pytest.raises(IndexError): - z.get_basic_selection(selection) + z.get_basic_selection(selection_bad) # type: ignore[arg-type] with pytest.raises(IndexError): - z[selection] + z[selection_bad] # type: ignore[index] with pytest.raises(IndexError): z.get_basic_selection([1, 0]) # type: ignore[arg-type] -basic_selections_2d = [ +basic_selections_2d: list[BasicSelection] = [ # single row 42, -1, @@ -340,9 +355,9 @@ def test_get_basic_selection_2d(store: StorePath) -> None: [0, 1], (slice(None), [0, 1]), ] - for selection in bad_selections: + for selection_bad in bad_selections: with pytest.raises(IndexError): - z.get_basic_selection(selection) + z.get_basic_selection(selection_bad) # type: ignore[arg-type] # check fallback on fancy indexing fancy_selection = ([0, 1], [0, 1]) np.testing.assert_array_equal(z[fancy_selection], [0, 11]) @@ -389,7 +404,7 @@ def test_fancy_indexing_fallback_on_get_setitem(store: StorePath) -> None: ], ) def test_orthogonal_indexing_fallback_on_getitem_2d( - store: StorePath, index, expected_result + store: StorePath, index: Selection, expected_result: npt.ArrayLike ) -> None: """ Tests the orthogonal indexing fallback on __getitem__ for a 2D matrix. @@ -407,6 +422,9 @@ def test_orthogonal_indexing_fallback_on_getitem_2d( np.testing.assert_array_equal(z[index], expected_result) +Index = list[int] | tuple[slice | int | list[int], ...] + + @pytest.mark.parametrize( "index,expected_result", [ @@ -421,7 +439,7 @@ def test_orthogonal_indexing_fallback_on_getitem_2d( ], ) def test_orthogonal_indexing_fallback_on_getitem_3d( - store: StorePath, index, expected_result + store: StorePath, index: Selection, expected_result: npt.ArrayLike ) -> None: """ Tests the orthogonal indexing fallback on __getitem__ for a 3D matrix. @@ -461,7 +479,7 @@ def test_orthogonal_indexing_fallback_on_getitem_3d( ], ) def test_orthogonal_indexing_fallback_on_setitem_2d( - store: StorePath, index, expected_result + store: StorePath, index: Selection, expected_result: npt.ArrayLike ) -> None: """ Tests the orthogonal indexing fallback on __setitem__ for a 3D matrix. @@ -485,9 +503,9 @@ def test_fancy_indexing_doesnt_mix_with_implicit_slicing(store: StorePath) -> No with pytest.raises(IndexError): np.testing.assert_array_equal(z2[[1, 2, 3], [1, 2, 3]], 0) with pytest.raises(IndexError): - z2[..., [1, 2, 3]] = 2 + z2[..., [1, 2, 3]] = 2 # type: ignore[index] with pytest.raises(IndexError): - np.testing.assert_array_equal(z2[..., [1, 2, 3]], 0) + np.testing.assert_array_equal(z2[..., [1, 2, 3]], 0) # type: ignore[index] @pytest.mark.parametrize( @@ -532,7 +550,9 @@ def test_set_basic_selection_0d( # arr_z[..., "foo", "bar"] = v[["foo", "bar"]] -def _test_get_orthogonal_selection(a, z, selection) -> None: +def _test_get_orthogonal_selection( + a: npt.NDArray[Any], z: Array, selection: OrthogonalSelection +) -> None: expect = oindex(a, selection) actual = z.get_orthogonal_selection(selection) assert_array_equal(expect, actual) @@ -558,7 +578,8 @@ def test_get_orthogonal_selection_1d_bool(store: StorePath) -> None: with pytest.raises(IndexError): z.oindex[np.zeros(2000, dtype=bool)] # too long with pytest.raises(IndexError): - z.oindex[[[True, False], [False, True]]] # too many dimensions + # too many dimensions + z.oindex[[[True, False], [False, True]]] # type: ignore[index] # noinspection PyStatementEffect @@ -594,14 +615,16 @@ def test_get_orthogonal_selection_1d_int(store: StorePath) -> None: [-(a.shape[0] + 1)], # out of bounds [[2, 4], [6, 8]], # too many dimensions ] - for selection in bad_selections: + for bad_selection in bad_selections: with pytest.raises(IndexError): - z.get_orthogonal_selection(selection) + z.get_orthogonal_selection(bad_selection) # type: ignore[arg-type] with pytest.raises(IndexError): - z.oindex[selection] + z.oindex[bad_selection] # type: ignore[index] -def _test_get_orthogonal_selection_2d(a, z, ix0, ix1) -> None: +def _test_get_orthogonal_selection_2d( + a: npt.NDArray[Any], z: Array, ix0: npt.NDArray[np.bool], ix1: npt.NDArray[np.bool] +) -> None: selections = [ # index both axes with array (ix0, ix1), @@ -651,17 +674,23 @@ def test_get_orthogonal_selection_2d(store: StorePath) -> None: ix1 = ix1[::-1] _test_get_orthogonal_selection_2d(a, z, ix0, ix1) - for selection in basic_selections_2d: - _test_get_orthogonal_selection(a, z, selection) + for selection_2d in basic_selections_2d: + _test_get_orthogonal_selection(a, z, selection_2d) - for selection in basic_selections_2d_bad: + for selection_2d_bad in basic_selections_2d_bad: with pytest.raises(IndexError): - z.get_orthogonal_selection(selection) + z.get_orthogonal_selection(selection_2d_bad) # type: ignore[arg-type] with pytest.raises(IndexError): - z.oindex[selection] + z.oindex[selection_2d_bad] # type: ignore[index] -def _test_get_orthogonal_selection_3d(a, z, ix0, ix1, ix2) -> None: +def _test_get_orthogonal_selection_3d( + a: npt.NDArray, + z: Array, + ix0: npt.NDArray[np.bool], + ix1: npt.NDArray[np.bool], + ix2: npt.NDArray[np.bool], +) -> None: selections = [ # single value (84, 42, 4), @@ -738,7 +767,9 @@ def test_orthogonal_indexing_edge_cases(store: StorePath) -> None: assert_array_equal(expect, actual) -def _test_set_orthogonal_selection(v, a, z, selection) -> None: +def _test_set_orthogonal_selection( + v: npt.NDArray[np.int_], a: npt.NDArray[Any], z: Array, selection: OrthogonalSelection +) -> None: for value in 42, oindex(v, selection), oindex(v, selection).tolist(): if isinstance(value, list) and value == []: # skip these cases as cannot preserve all dimensions @@ -782,7 +813,13 @@ def test_set_orthogonal_selection_1d(store: StorePath) -> None: _test_set_orthogonal_selection(v, a, z, selection) -def _test_set_orthogonal_selection_2d(v, a, z, ix0, ix1) -> None: +def _test_set_orthogonal_selection_2d( + v: npt.NDArray[np.int_], + a: npt.NDArray[np.int_], + z: Array, + ix0: npt.NDArray[np.bool], + ix1: npt.NDArray[np.bool], +) -> None: selections = [ # index both axes with array (ix0, ix1), @@ -825,7 +862,14 @@ def test_set_orthogonal_selection_2d(store: StorePath) -> None: _test_set_orthogonal_selection(v, a, z, selection) -def _test_set_orthogonal_selection_3d(v, a, z, ix0, ix1, ix2) -> None: +def _test_set_orthogonal_selection_3d( + v: npt.NDArray[np.int_], + a: npt.NDArray[np.int_], + z: Array, + ix0: npt.NDArray[np.bool], + ix1: npt.NDArray[np.bool], + ix2: npt.NDArray[np.bool], +) -> None: selections = ( # single value (84, 42, 4), @@ -906,7 +950,9 @@ def test_orthogonal_indexing_fallback_on_get_setitem(store: StorePath) -> None: np.testing.assert_array_equal(z2[:], [0, 1, 1, 1, 0]) -def _test_get_coordinate_selection(a, z, selection) -> None: +def _test_get_coordinate_selection( + a: npt.NDArray, z: Array, selection: CoordinateSelection +) -> None: expect = a[selection] actual = z.get_coordinate_selection(selection) assert_array_equal(expect, actual) @@ -967,9 +1013,9 @@ def test_get_coordinate_selection_1d(store: StorePath) -> None: ] for selection in bad_selections: with pytest.raises(IndexError): - z.get_coordinate_selection(selection) + z.get_coordinate_selection(selection) # type: ignore[arg-type] with pytest.raises(IndexError): - z.vindex[selection] + z.vindex[selection] # type: ignore[index] def test_get_coordinate_selection_2d(store: StorePath) -> None: @@ -978,6 +1024,8 @@ def test_get_coordinate_selection_2d(store: StorePath) -> None: z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) np.random.seed(42) + ix0: npt.ArrayLike + ix1: npt.ArrayLike # test with different degrees of sparseness for p in 2, 0.5, 0.1, 0.01: n = int(a.size * p) @@ -1014,19 +1062,21 @@ def test_get_coordinate_selection_2d(store: StorePath) -> None: with pytest.raises(IndexError): selection = slice(5, 15), [1, 2, 3] - z.get_coordinate_selection(selection) + z.get_coordinate_selection(selection) # type:ignore[arg-type] with pytest.raises(IndexError): selection = [1, 2, 3], slice(5, 15) - z.get_coordinate_selection(selection) + z.get_coordinate_selection(selection) # type:ignore[arg-type] with pytest.raises(IndexError): selection = Ellipsis, [1, 2, 3] - z.get_coordinate_selection(selection) + z.get_coordinate_selection(selection) # type:ignore[arg-type] with pytest.raises(IndexError): selection = Ellipsis - z.get_coordinate_selection(selection) + z.get_coordinate_selection(selection) # type:ignore[arg-type] -def _test_set_coordinate_selection(v, a, z, selection) -> None: +def _test_set_coordinate_selection( + v: npt.NDArray, a: npt.NDArray, z: Array, selection: CoordinateSelection +) -> None: for value in 42, v[selection], v[selection].tolist(): # setup expectation a[:] = 0 @@ -1060,9 +1110,9 @@ def test_set_coordinate_selection_1d(store: StorePath) -> None: for selection in coordinate_selections_1d_bad: with pytest.raises(IndexError): - z.set_coordinate_selection(selection, 42) + z.set_coordinate_selection(selection, 42) # type:ignore[arg-type] with pytest.raises(IndexError): - z.vindex[selection] = 42 + z.vindex[selection] = 42 # type:ignore[index] def test_set_coordinate_selection_2d(store: StorePath) -> None: @@ -1096,7 +1146,12 @@ def test_set_coordinate_selection_2d(store: StorePath) -> None: _test_set_coordinate_selection(v, a, z, (ix0, ix1)) -def _test_get_block_selection(a, z, selection, expected_idx) -> None: +def _test_get_block_selection( + a: npt.NDArray[Any], + z: Array, + selection: BasicSelection, + expected_idx: slice | tuple[slice, ...], +) -> None: expect = a[expected_idx] actual = z.get_block_selection(selection) assert_array_equal(expect, actual) @@ -1104,7 +1159,7 @@ def _test_get_block_selection(a, z, selection, expected_idx) -> None: assert_array_equal(expect, actual) -block_selections_1d = [ +block_selections_1d: list[BasicSelection] = [ # test single item 0, 5, @@ -1119,7 +1174,7 @@ def _test_get_block_selection(a, z, selection, expected_idx) -> None: slice(None), # Full slice ] -block_selections_1d_array_projection = [ +block_selections_1d_array_projection: list[slice] = [ # test single item slice(100), slice(500, 600), @@ -1163,14 +1218,14 @@ def test_get_block_selection_1d(store: StorePath) -> None: -(z.metadata.chunk_grid.get_nchunks(z.shape) + 1), # out of bounds ] - for selection in bad_selections: + for selection_bad in bad_selections: with pytest.raises(IndexError): - z.get_block_selection(selection) + z.get_block_selection(selection_bad) # type:ignore[arg-type] with pytest.raises(IndexError): - z.blocks[selection] + z.blocks[selection_bad] # type:ignore[index] -block_selections_2d = [ +block_selections_2d: list[BasicSelection] = [ # test single item (0, 0), (1, 2), @@ -1185,7 +1240,7 @@ def test_get_block_selection_1d(store: StorePath) -> None: (slice(None), slice(None)), # Full slice ] -block_selections_2d_array_projection = [ +block_selections_2d_array_projection: list[tuple[slice, slice]] = [ # test single item (slice(300), slice(3)), (slice(300, 600), slice(6, 9)), @@ -1223,7 +1278,11 @@ def test_get_block_selection_2d(store: StorePath) -> None: def _test_set_block_selection( - v: np.ndarray, a: np.ndarray, z: zarr.Array, selection, expected_idx + v: npt.NDArray[Any], + a: npt.NDArray[Any], + z: zarr.Array, + selection: BasicSelection, + expected_idx: slice, ) -> None: for value in 42, v[expected_idx], v[expected_idx].tolist(): # setup expectation @@ -1250,11 +1309,11 @@ def test_set_block_selection_1d(store: StorePath) -> None: ): _test_set_block_selection(v, a, z, selection, expected_idx) - for selection in block_selections_1d_bad: + for selection_bad in block_selections_1d_bad: with pytest.raises(IndexError): - z.set_block_selection(selection, 42) + z.set_block_selection(selection_bad, 42) # type:ignore[arg-type] with pytest.raises(IndexError): - z.blocks[selection] = 42 + z.blocks[selection_bad] = 42 # type:ignore[index] def test_set_block_selection_2d(store: StorePath) -> None: @@ -1279,7 +1338,7 @@ def test_set_block_selection_2d(store: StorePath) -> None: z.set_block_selection(selection, 42) -def _test_get_mask_selection(a, z, selection) -> None: +def _test_get_mask_selection(a: npt.NDArray[Any], z: Array, selection: npt.NDArray) -> None: expect = a[selection] actual = z.get_mask_selection(selection) assert_array_equal(expect, actual) @@ -1324,9 +1383,9 @@ def test_get_mask_selection_1d(store: StorePath) -> None: ] for selection in bad_selections: with pytest.raises(IndexError): - z.get_mask_selection(selection) + z.get_mask_selection(selection) # type: ignore[arg-type] with pytest.raises(IndexError): - z.vindex[selection] + z.vindex[selection] # type:ignore[index] # noinspection PyStatementEffect @@ -1350,7 +1409,9 @@ def test_get_mask_selection_2d(store: StorePath) -> None: z.vindex[[True, False]] # wrong no. dimensions -def _test_set_mask_selection(v, a, z, selection) -> None: +def _test_set_mask_selection( + v: npt.NDArray, a: npt.NDArray, z: Array, selection: npt.NDArray +) -> None: a[:] = 0 z[:] = 0 a[selection] = v[selection] @@ -1378,9 +1439,9 @@ def test_set_mask_selection_1d(store: StorePath) -> None: for selection in mask_selections_1d_bad: with pytest.raises(IndexError): - z.set_mask_selection(selection, 42) + z.set_mask_selection(selection, 42) # type: ignore[arg-type] with pytest.raises(IndexError): - z.vindex[selection] = 42 + z.vindex[selection] = 42 # type: ignore[index] def test_set_mask_selection_2d(store: StorePath) -> None: @@ -1413,7 +1474,7 @@ def test_get_selection_out(store: StorePath) -> None: assert_array_equal(expect, out.as_numpy_array()[:]) with pytest.raises(TypeError): - z.get_basic_selection(Ellipsis, out=[]) + z.get_basic_selection(Ellipsis, out=[]) # type: ignore[arg-type] # orthogonal selections a = np.arange(10000, dtype=int).reshape(1000, 10) @@ -1467,11 +1528,13 @@ def test_get_selection_out(store: StorePath) -> None: @pytest.mark.xfail(reason="fields are not supported in v3") def test_get_selections_with_fields(store: StorePath) -> None: - a = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] - a = np.array(a, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) + a = np.array( + [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)], + dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], + ) z = zarr_array_from_numpy_array(store, a, chunk_shape=(2,)) - fields_fixture = [ + fields_fixture: list[str | list[str]] = [ "foo", ["foo"], ["foo", "bar"], @@ -1568,17 +1631,19 @@ def test_get_selections_with_fields(store: StorePath) -> None: with pytest.raises(IndexError): z.get_basic_selection(Ellipsis, fields=["notafield"]) with pytest.raises(IndexError): - z.get_basic_selection(Ellipsis, fields=slice(None)) + z.get_basic_selection(Ellipsis, fields=slice(None)) # type: ignore[arg-type] @pytest.mark.xfail(reason="fields are not supported in v3") def test_set_selections_with_fields(store: StorePath) -> None: - v = [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)] - v = np.array(v, dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")]) + v = np.array( + [("aaa", 1, 4.2), ("bbb", 2, 8.4), ("ccc", 3, 12.6)], + dtype=[("foo", "S3"), ("bar", "i4"), ("baz", "f8")], + ) a = np.empty_like(v) z = zarr_array_from_numpy_array(store, v, chunk_shape=(2,)) - fields_fixture = [ + fields_fixture: list[str | list[str]] = [ "foo", [], ["foo"], @@ -1597,11 +1662,11 @@ def test_set_selections_with_fields(store: StorePath) -> None: with pytest.raises(IndexError): z.set_basic_selection(Ellipsis, v, fields=fields) with pytest.raises(IndexError): - z.set_orthogonal_selection([0, 2], v, fields=fields) + z.set_orthogonal_selection([0, 2], v, fields=fields) # type: ignore[arg-type] with pytest.raises(IndexError): z.set_coordinate_selection([0, 2], v, fields=fields) with pytest.raises(IndexError): - z.set_mask_selection([True, False, True], v, fields=fields) + z.set_mask_selection([True, False, True], v, fields=fields) # type: ignore[arg-type] else: if isinstance(fields, list) and len(fields) == 1: @@ -1694,7 +1759,9 @@ def test_numpy_int_indexing(store: StorePath) -> None: ), ], ) -async def test_accessed_chunks(shape, chunks, ops) -> None: +async def test_accessed_chunks( + shape: tuple[int, ...], chunks: tuple[int, ...], ops: list[tuple[str, tuple[slice, ...]]] +) -> None: # Test that only the required chunks are accessed during basic selection operations # shape: array shape # chunks: chunk size @@ -1767,7 +1834,7 @@ async def test_accessed_chunks(shape, chunks, ops) -> None: [[100, 200, 300], [4, 5, 6]], ], ) -def test_indexing_equals_numpy(store, selection) -> None: +def test_indexing_equals_numpy(store: StorePath, selection: Selection) -> None: a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) # note: in python 3.10 a[*selection] is not valid unpacking syntax @@ -1785,7 +1852,9 @@ def test_indexing_equals_numpy(store, selection) -> None: [np.full(1000, True), [True, False] * 5], ], ) -def test_orthogonal_bool_indexing_like_numpy_ix(store, selection) -> None: +def test_orthogonal_bool_indexing_like_numpy_ix( + store: StorePath, selection: list[npt.ArrayLike] +) -> None: a = np.arange(10000, dtype=int).reshape(1000, 10) z = zarr_array_from_numpy_array(store, a, chunk_shape=(300, 3)) expected = a[np.ix_(*selection)] From c878da2a900fc621ff23cc6d84d45cd3cb26cbed Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Thu, 19 Sep 2024 19:52:53 -0500 Subject: [PATCH 0168/1043] Make Group.arrays, groups compatible with v2 (#2213) Defines a set of array / group iterators. - .groups / .arrays: over (name, value) pairs - .group_keys / .array_keys: over keys - .group_values / .array_values: over values Co-authored-by: Joe Hamman --- src/zarr/core/group.py | 67 +++++++++++++++++++++++++----------------- tests/v3/test_group.py | 44 ++++++++++++--------------- 2 files changed, 59 insertions(+), 52 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index a4e1e252ea..a45c7e1df7 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -33,7 +33,7 @@ from zarr.store.common import ensure_no_existing_node if TYPE_CHECKING: - from collections.abc import AsyncGenerator, Iterable, Iterator + from collections.abc import AsyncGenerator, Generator, Iterable, Iterator from typing import Any from zarr.abc.codec import Codec @@ -678,29 +678,31 @@ async def contains(self, member: str) -> bool: else: return True - # todo: decide if this method should be separate from `groups` - async def group_keys(self) -> AsyncGenerator[str, None]: - async for key, value in self.members(): + async def groups(self) -> AsyncGenerator[tuple[str, AsyncGroup], None]: + async for name, value in self.members(): if isinstance(value, AsyncGroup): - yield key + yield name, value - # todo: decide if this method should be separate from `group_keys` - async def groups(self) -> AsyncGenerator[AsyncGroup, None]: - async for _, value in self.members(): - if isinstance(value, AsyncGroup): - yield value + async def group_keys(self) -> AsyncGenerator[str, None]: + async for key, _ in self.groups(): + yield key - # todo: decide if this method should be separate from `arrays` - async def array_keys(self) -> AsyncGenerator[str, None]: + async def group_values(self) -> AsyncGenerator[AsyncGroup, None]: + async for _, group in self.groups(): + yield group + + async def arrays(self) -> AsyncGenerator[tuple[str, AsyncArray], None]: async for key, value in self.members(): if isinstance(value, AsyncArray): - yield key + yield key, value - # todo: decide if this method should be separate from `array_keys` - async def arrays(self) -> AsyncGenerator[AsyncArray, None]: - async for _, value in self.members(): - if isinstance(value, AsyncArray): - yield value + async def array_keys(self) -> AsyncGenerator[str, None]: + async for key, _ in self.arrays(): + yield key + + async def array_values(self) -> AsyncGenerator[AsyncArray, None]: + async for _, array in self.arrays(): + yield array async def tree(self, expand: bool = False, level: int | None = None) -> Any: raise NotImplementedError @@ -861,18 +863,29 @@ def members(self, max_depth: int | None = 0) -> tuple[tuple[str, Array | Group], def __contains__(self, member: str) -> bool: return self._sync(self._async_group.contains(member)) - def group_keys(self) -> tuple[str, ...]: - return tuple(self._sync_iter(self._async_group.group_keys())) + def groups(self) -> Generator[tuple[str, Group], None]: + for name, async_group in self._sync_iter(self._async_group.groups()): + yield name, Group(async_group) + + def group_keys(self) -> Generator[str, None]: + for name, _ in self.groups(): + yield name + + def group_values(self) -> Generator[Group, None]: + for _, group in self.groups(): + yield group - def groups(self) -> tuple[Group, ...]: - # TODO: in v2 this was a generator that return key: Group - return tuple(Group(obj) for obj in self._sync_iter(self._async_group.groups())) + def arrays(self) -> Generator[tuple[str, Array], None]: + for name, async_array in self._sync_iter(self._async_group.arrays()): + yield name, Array(async_array) - def array_keys(self) -> tuple[str, ...]: - return tuple(self._sync_iter(self._async_group.array_keys())) + def array_keys(self) -> Generator[str, None]: + for name, _ in self.arrays(): + yield name - def arrays(self) -> tuple[Array, ...]: - return tuple(Array(obj) for obj in self._sync_iter(self._async_group.arrays())) + def array_values(self) -> Generator[Array, None]: + for _, array in self.arrays(): + yield array def tree(self, expand: bool = False, level: int | None = None) -> Any: return self._sync(self._async_group.tree(expand=expand, level=level)) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 4bb23fddaa..6e75294b7c 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -301,34 +301,28 @@ def test_group_contains(store: Store, zarr_format: ZarrFormat) -> None: assert "foo" in group -def test_group_subgroups(store: Store, zarr_format: ZarrFormat) -> None: - """ - Test the behavior of `Group` methods for accessing subgroups, namely `Group.group_keys` and `Group.groups` - """ +def test_group_child_iterators(store: Store, zarr_format: ZarrFormat): group = Group.create(store, zarr_format=zarr_format) - keys = ("foo", "bar") - subgroups_expected = tuple(group.create_group(k) for k in keys) - # create a sub-array as well - _ = group.create_array("array", shape=(10,)) - subgroups_observed = group.groups() - assert set(group.group_keys()) == set(keys) - assert len(subgroups_observed) == len(subgroups_expected) - assert all(a in subgroups_observed for a in subgroups_expected) + expected_group_keys = ["g0", "g1"] + expected_group_values = [group.create_group(name=name) for name in expected_group_keys] + expected_groups = list(zip(expected_group_keys, expected_group_values, strict=False)) + expected_group_values[0].create_group("subgroup") + expected_group_values[0].create_array("subarray", shape=(1,)) -def test_group_subarrays(store: Store, zarr_format: ZarrFormat) -> None: - """ - Test the behavior of `Group` methods for accessing subgroups, namely `Group.group_keys` and `Group.groups` - """ - group = Group.create(store, zarr_format=zarr_format) - keys = ("foo", "bar") - subarrays_expected = tuple(group.create_array(k, shape=(10,)) for k in keys) - # create a sub-group as well - _ = group.create_group("group") - subarrays_observed = group.arrays() - assert set(group.array_keys()) == set(keys) - assert len(subarrays_observed) == len(subarrays_expected) - assert all(a in subarrays_observed for a in subarrays_expected) + expected_array_keys = ["a0", "a1"] + expected_array_values = [ + group.create_array(name=name, shape=(1,)) for name in expected_array_keys + ] + expected_arrays = list(zip(expected_array_keys, expected_array_values, strict=False)) + + assert sorted(group.groups(), key=lambda x: x[0]) == expected_groups + assert sorted(group.group_keys()) == expected_group_keys + assert sorted(group.group_values(), key=lambda x: x.name) == expected_group_values + + assert sorted(group.arrays(), key=lambda x: x[0]) == expected_arrays + assert sorted(group.array_keys()) == expected_array_keys + assert sorted(group.array_values(), key=lambda x: x.name) == expected_array_values def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: From 6900754138e37c75f952233996d80acbedad60f3 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Fri, 20 Sep 2024 09:45:01 -0500 Subject: [PATCH 0169/1043] Basic Zarr-python 2.x compatibility changes (#2098) * WIP - backwards compat * fixup put * rm consolidated * typing fixup * revert unneded change * fixup * deprecate positional args * attribute * Fixup * fixup * fixup * fixup * fixup * fixup * fixup * fixup * fixup * fixup * fixup * fixup * fixup * ci * fixup * fixup --------- Co-authored-by: Joe Hamman --- pyproject.toml | 3 +- src/zarr/_compat.py | 68 ++++++++++++++++++++++++++ src/zarr/api/asynchronous.py | 6 +-- src/zarr/api/synchronous.py | 11 +++-- src/zarr/core/array.py | 13 +++++ src/zarr/core/attributes.py | 16 +++++++ src/zarr/core/group.py | 45 +++++++++++++++-- src/zarr/testing/strategies.py | 2 +- tests/v3/conftest.py | 2 +- tests/v3/test_api.py | 39 ++++++++++++++- tests/v3/test_array.py | 46 +++++++++++++++++- tests/v3/test_attributes.py | 13 +++++ tests/v3/test_buffer.py | 4 ++ tests/v3/test_group.py | 88 +++++++++++++++++----------------- tests/v3/test_sync.py | 8 ++++ 15 files changed, 303 insertions(+), 61 deletions(-) create mode 100644 src/zarr/_compat.py create mode 100644 tests/v3/test_attributes.py diff --git a/pyproject.toml b/pyproject.toml index 886cd5a0bc..41c2006b2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,8 @@ test = [ "flask", "requests", "mypy", - "hypothesis" + "hypothesis", + "universal-pathlib", ] jupyter = [ diff --git a/src/zarr/_compat.py b/src/zarr/_compat.py new file mode 100644 index 0000000000..52d96005cc --- /dev/null +++ b/src/zarr/_compat.py @@ -0,0 +1,68 @@ +import warnings +from collections.abc import Callable +from functools import wraps +from inspect import Parameter, signature +from typing import Any, TypeVar + +T = TypeVar("T") + +# Based off https://github.com/scikit-learn/scikit-learn/blob/e87b32a81c70abed8f2e97483758eb64df8255e9/sklearn/utils/validation.py#L63 + + +def _deprecate_positional_args( + func: Callable[..., T] | None = None, *, version: str = "3.1.0" +) -> Callable[..., T]: + """Decorator for methods that issues warnings for positional arguments. + + Using the keyword-only argument syntax in pep 3102, arguments after the + * will issue a warning when passed as a positional argument. + + Parameters + ---------- + func : callable, default=None + Function to check arguments on. + version : callable, default="3.1.0" + The version when positional arguments will result in error. + """ + + def _inner_deprecate_positional_args(f: Callable[..., T]) -> Callable[..., T]: + sig = signature(f) + kwonly_args = [] + all_args = [] + + for name, param in sig.parameters.items(): + if param.kind == Parameter.POSITIONAL_OR_KEYWORD: + all_args.append(name) + elif param.kind == Parameter.KEYWORD_ONLY: + kwonly_args.append(name) + + @wraps(f) + def inner_f(*args: Any, **kwargs: Any) -> T: + extra_args = len(args) - len(all_args) + if extra_args <= 0: + return f(*args, **kwargs) + + # extra_args > 0 + args_msg = [ + f"{name}={arg}" + for name, arg in zip(kwonly_args[:extra_args], args[-extra_args:], strict=False) + ] + formatted_args_msg = ", ".join(args_msg) + warnings.warn( + ( + f"Pass {formatted_args_msg} as keyword args. From version " + f"{version} passing these as positional arguments " + "will result in an error" + ), + FutureWarning, + stacklevel=2, + ) + kwargs.update(zip(sig.parameters, args, strict=False)) + return f(**kwargs) + + return inner_f + + if func is not None: + return _inner_deprecate_positional_args(func) + + return _inner_deprecate_positional_args # type: ignore[return-value] diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 58278172f7..5fbb38c5e7 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -503,7 +503,7 @@ async def group( try: return await AsyncGroup.open(store=store_path, zarr_format=zarr_format) except (KeyError, FileNotFoundError): - return await AsyncGroup.create( + return await AsyncGroup.from_store( store=store_path, zarr_format=zarr_format or _default_zarr_version(), exists_ok=overwrite, @@ -512,8 +512,8 @@ async def group( async def open_group( - *, # Note: this is a change from v2 store: StoreLike | None = None, + *, # Note: this is a change from v2 mode: AccessModeLiteral | None = None, cache_attrs: bool | None = None, # not used, default changed synchronizer: Any = None, # not used @@ -590,7 +590,7 @@ async def open_group( try: return await AsyncGroup.open(store_path, zarr_format=zarr_format) except (KeyError, FileNotFoundError): - return await AsyncGroup.create( + return await AsyncGroup.from_store( store_path, zarr_format=zarr_format or _default_zarr_version(), exists_ok=True, diff --git a/src/zarr/api/synchronous.py b/src/zarr/api/synchronous.py index 129f901934..bc4a7bfafd 100644 --- a/src/zarr/api/synchronous.py +++ b/src/zarr/api/synchronous.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any import zarr.api.asynchronous as async_api +from zarr._compat import _deprecate_positional_args from zarr.core.array import Array, AsyncArray from zarr.core.group import Group from zarr.core.sync import sync @@ -63,9 +64,10 @@ def load( return sync(async_api.load(store=store, zarr_version=zarr_version, path=path)) +@_deprecate_positional_args def open( - *, store: StoreLike | None = None, + *, mode: AccessModeLiteral | None = None, # type and value changed zarr_version: ZarrFormat | None = None, # deprecated zarr_format: ZarrFormat | None = None, @@ -107,6 +109,7 @@ def save( ) +@_deprecate_positional_args def save_array( store: StoreLike, arr: NDArrayLike, @@ -159,9 +162,10 @@ def array(data: NDArrayLike, **kwargs: Any) -> Array: return Array(sync(async_api.array(data=data, **kwargs))) +@_deprecate_positional_args def group( - *, # Note: this is a change from v2 store: StoreLike | None = None, + *, # Note: this is a change from v2 overwrite: bool = False, chunk_store: StoreLike | None = None, # not used in async_api cache_attrs: bool | None = None, # default changed, not used in async_api @@ -190,9 +194,10 @@ def group( ) +@_deprecate_positional_args def open_group( - *, # Note: this is a change from v2 store: StoreLike | None = None, + *, # Note: this is a change from v2 mode: AccessModeLiteral | None = None, # not used in async api cache_attrs: bool | None = None, # default changed, not used in async api synchronizer: Any = None, # not used in async api diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index 2cf6d69404..b825ca4ca1 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -8,6 +8,8 @@ import numpy as np import numpy.typing as npt +from zarr._compat import _deprecate_positional_args +from zarr.abc.codec import Codec, CodecPipeline from zarr.abc.store import set_or_delete from zarr.codecs import BytesCodec from zarr.codecs._v2 import V2Compressor, V2Filters @@ -621,6 +623,7 @@ class Array: _async_array: AsyncArray @classmethod + @_deprecate_positional_args def create( cls, store: StoreLike, @@ -1016,6 +1019,7 @@ def __setitem__(self, selection: Selection, value: npt.ArrayLike) -> None: else: self.set_basic_selection(cast(BasicSelection, pure_selection), value, fields=fields) + @_deprecate_positional_args def get_basic_selection( self, selection: BasicSelection = Ellipsis, @@ -1139,6 +1143,7 @@ def get_basic_selection( ) ) + @_deprecate_positional_args def set_basic_selection( self, selection: BasicSelection, @@ -1234,6 +1239,7 @@ def set_basic_selection( indexer = BasicIndexer(selection, self.shape, self.metadata.chunk_grid) sync(self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) + @_deprecate_positional_args def get_orthogonal_selection( self, selection: OrthogonalSelection, @@ -1358,6 +1364,7 @@ def get_orthogonal_selection( ) ) + @_deprecate_positional_args def set_orthogonal_selection( self, selection: OrthogonalSelection, @@ -1468,6 +1475,7 @@ def set_orthogonal_selection( self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype) ) + @_deprecate_positional_args def get_mask_selection( self, mask: MaskSelection, @@ -1550,6 +1558,7 @@ def get_mask_selection( ) ) + @_deprecate_positional_args def set_mask_selection( self, mask: MaskSelection, @@ -1628,6 +1637,7 @@ def set_mask_selection( indexer = MaskIndexer(mask, self.shape, self.metadata.chunk_grid) sync(self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) + @_deprecate_positional_args def get_coordinate_selection( self, selection: CoordinateSelection, @@ -1717,6 +1727,7 @@ def get_coordinate_selection( out_array = np.array(out_array).reshape(indexer.sel_shape) return out_array + @_deprecate_positional_args def set_coordinate_selection( self, selection: CoordinateSelection, @@ -1806,6 +1817,7 @@ def set_coordinate_selection( sync(self._async_array._set_selection(indexer, value, fields=fields, prototype=prototype)) + @_deprecate_positional_args def get_block_selection( self, selection: BasicSelection, @@ -1904,6 +1916,7 @@ def get_block_selection( ) ) + @_deprecate_positional_args def set_block_selection( self, selection: BasicSelection, diff --git a/src/zarr/core/attributes.py b/src/zarr/core/attributes.py index 09677f7bdc..62ff5fc935 100644 --- a/src/zarr/core/attributes.py +++ b/src/zarr/core/attributes.py @@ -35,3 +35,19 @@ def __iter__(self) -> Iterator[str]: def __len__(self) -> int: return len(self._obj.metadata.attributes) + + def put(self, d: dict[str, JSON]) -> None: + """ + Overwrite all attributes with the values from `d`. + + Equivalent to the following pseudo-code, but performed atomically. + + .. code-block:: python + + >>> attrs = {"a": 1, "b": 2} + >>> attrs.clear() + >>> attrs.update({"a": 3", "c": 4}) + >>> attrs + {'a': 3, 'c': 4} + """ + self._obj = self._obj.update_attributes(d) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index a45c7e1df7..7c56707a4f 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -12,7 +12,7 @@ import zarr.api.asynchronous as async_api from zarr.abc.metadata import Metadata -from zarr.abc.store import set_or_delete +from zarr.abc.store import Store, set_or_delete from zarr.core.array import Array, AsyncArray from zarr.core.attributes import Attributes from zarr.core.buffer import default_buffer_prototype @@ -126,7 +126,7 @@ class AsyncGroup: store_path: StorePath @classmethod - async def create( + async def from_store( cls, store: StoreLike, *, @@ -312,6 +312,21 @@ def attrs(self) -> dict[str, Any]: def info(self) -> None: raise NotImplementedError + @property + def store(self) -> Store: + return self.store_path.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self.store_path.store.mode.readonly + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return None + async def create_group( self, name: str, @@ -320,7 +335,7 @@ async def create_group( attributes: dict[str, Any] | None = None, ) -> AsyncGroup: attributes = attributes or {} - return await type(self).create( + return await type(self).from_store( self.store_path / name, attributes=attributes, exists_ok=exists_ok, @@ -752,7 +767,7 @@ class Group(SyncMixin): _async_group: AsyncGroup @classmethod - def create( + def from_store( cls, store: StoreLike, *, @@ -762,7 +777,7 @@ def create( ) -> Group: attributes = attributes or {} obj = sync( - AsyncGroup.create( + AsyncGroup.from_store( store, attributes=attributes, exists_ok=exists_ok, @@ -843,6 +858,22 @@ def attrs(self) -> Attributes: def info(self) -> None: raise NotImplementedError + @property + def store(self) -> Store: + # Backwards compatibility for 2.x + return self._async_group.store + + @property + def read_only(self) -> bool: + # Backwards compatibility for 2.x + return self._async_group.read_only + + @property + def synchronizer(self) -> None: + # Backwards compatibility for 2.x + # Not implemented in 3.x yet. + return self._async_group.synchronizer + def update_attributes(self, new_attributes: dict[str, Any]) -> Group: self._sync(self._async_group.update_attributes(new_attributes)) return self @@ -913,6 +944,10 @@ def require_groups(self, *names: str) -> tuple[Group, ...]: """Convenience method to require multiple groups in a single call.""" return tuple(map(Group, self._sync(self._async_group.require_groups(*names)))) + def create(self, *args: Any, **kwargs: Any) -> Array: + # Backwards compatibility for 2.x + return self.create_array(*args, **kwargs) + def create_array( self, name: str, diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 83de3d92ce..2f4ca8a697 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -99,7 +99,7 @@ def arrays( expected_attrs = {} if attributes is None else attributes array_path = path + ("/" if not path.endswith("/") else "") + name - root = Group.create(store) + root = Group.from_store(store) fill_value_args: tuple[Any, ...] = tuple() if nparray.dtype.kind == "M": m = re.search(r"\[(.+)\]", nparray.dtype.str) diff --git a/tests/v3/conftest.py b/tests/v3/conftest.py index 41cd359346..d1ac410753 100644 --- a/tests/v3/conftest.py +++ b/tests/v3/conftest.py @@ -89,7 +89,7 @@ async def async_group(request: pytest.FixtureRequest, tmpdir: LEGACY_PATH) -> As param: AsyncGroupRequest = request.param store = await parse_store(param.store, str(tmpdir)) - agroup = await AsyncGroup.create( + agroup = await AsyncGroup.from_store( store, attributes=param.attributes, zarr_format=param.zarr_format, diff --git a/tests/v3/test_api.py b/tests/v3/test_api.py index ddfab587cc..728638c871 100644 --- a/tests/v3/test_api.py +++ b/tests/v3/test_api.py @@ -1,4 +1,5 @@ import pathlib +import warnings import numpy as np import pytest @@ -7,7 +8,7 @@ import zarr from zarr import Array, Group from zarr.abc.store import Store -from zarr.api.synchronous import create, load, open, open_group, save, save_array, save_group +from zarr.api.synchronous import create, group, load, open, open_group, save, save_array, save_group from zarr.core.common import ZarrFormat from zarr.store.memory import MemoryStore @@ -108,7 +109,7 @@ def test_save_errors() -> None: save_group("data/group.zarr") with pytest.raises(TypeError): # no array provided - save_array("data/group.zarr") # type: ignore[call-arg] + save_array("data/group.zarr") with pytest.raises(ValueError): # no arrays provided save("data/group.zarr") @@ -878,3 +879,37 @@ def test_tree() -> None: # # bad option # with pytest.raises(TypeError): # copy(source["foo"], dest, dry_run=True, log=True) + + +def test_open_positional_args_deprecated() -> None: + store = MemoryStore({}, mode="w") + with pytest.warns(FutureWarning, match="pass"): + open(store, "w", shape=(1,)) + + +def test_save_array_positional_args_deprecated() -> None: + store = MemoryStore({}, mode="w") + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", message="zarr_version is deprecated", category=DeprecationWarning + ) + with pytest.warns(FutureWarning, match="pass"): + save_array( + store, + np.ones( + 1, + ), + 3, + ) + + +def test_group_positional_args_deprecated() -> None: + store = MemoryStore({}, mode="w") + with pytest.warns(FutureWarning, match="pass"): + group(store, True) + + +def test_open_group_positional_args_deprecated() -> None: + store = MemoryStore({}, mode="w") + with pytest.warns(FutureWarning, match="pass"): + open_group(store, "w") diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index b7beb63b1c..fe5c782a1b 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -5,6 +5,7 @@ import pytest from zarr import Array, AsyncArray, Group +from zarr.core.buffer.cpu import NDBuffer from zarr.core.common import ZarrFormat from zarr.errors import ContainsArrayError, ContainsGroupError from zarr.store import LocalStore, MemoryStore @@ -25,7 +26,7 @@ def test_array_creation_existing_node( Check that an existing array or group is handled as expected during array creation. """ spath = StorePath(store) - group = Group.create(spath, zarr_format=zarr_format) + group = Group.from_store(spath, zarr_format=zarr_format) expected_exception: type[ContainsArrayError] | type[ContainsGroupError] if extant_node == "array": expected_exception = ContainsArrayError @@ -76,7 +77,7 @@ def test_array_name_properties_no_group( def test_array_name_properties_with_group( store: LocalStore | MemoryStore, zarr_format: ZarrFormat ) -> None: - root = Group.create(store=store, zarr_format=zarr_format) + root = Group.from_store(store=store, zarr_format=zarr_format) foo = root.create_array("foo", shape=(100,), chunks=(10,), dtype="i4") assert foo.path == "foo" assert foo.name == "/foo" @@ -138,6 +139,47 @@ def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str assert arr.fill_value.dtype == arr.dtype +def test_create_positional_args_deprecated() -> None: + store = MemoryStore({}, mode="w") + with pytest.warns(FutureWarning, match="Pass"): + Array.create(store, (2, 2), dtype="f8") + + +def test_selection_positional_args_deprecated() -> None: + store = MemoryStore({}, mode="w") + arr = Array.create(store, shape=(2, 2), dtype="f8") + + with pytest.warns(FutureWarning, match="Pass out"): + arr.get_basic_selection(..., NDBuffer(array=np.empty((2, 2)))) + + with pytest.warns(FutureWarning, match="Pass fields"): + arr.set_basic_selection(..., 1, None) + + with pytest.warns(FutureWarning, match="Pass out"): + arr.get_orthogonal_selection(..., NDBuffer(array=np.empty((2, 2)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_orthogonal_selection(..., 1, None) + + with pytest.warns(FutureWarning, match="Pass"): + arr.get_mask_selection(np.zeros((2, 2), dtype=bool), NDBuffer(array=np.empty((0,)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_mask_selection(np.zeros((2, 2), dtype=bool), 1, None) + + with pytest.warns(FutureWarning, match="Pass"): + arr.get_coordinate_selection(([0, 1], [0, 1]), NDBuffer(array=np.empty((2,)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_coordinate_selection(([0, 1], [0, 1]), 1, None) + + with pytest.warns(FutureWarning, match="Pass"): + arr.get_block_selection((0, slice(None)), NDBuffer(array=np.empty((2, 2)))) + + with pytest.warns(FutureWarning, match="Pass"): + arr.set_block_selection((0, slice(None)), 1, None) + + @pytest.mark.parametrize("store", ["memory"], indirect=True) async def test_array_v3_nan_fill_value(store: MemoryStore) -> None: shape = (10,) diff --git a/tests/v3/test_attributes.py b/tests/v3/test_attributes.py new file mode 100644 index 0000000000..65b6a02e8d --- /dev/null +++ b/tests/v3/test_attributes.py @@ -0,0 +1,13 @@ +import zarr.core +import zarr.core.attributes +import zarr.store + + +def test_put() -> None: + store = zarr.store.MemoryStore({}, mode="w") + attrs = zarr.core.attributes.Attributes( + zarr.Group.from_store(store, attributes={"a": 1, "b": 2}) + ) + attrs.put({"a": 3, "c": 4}) + expected = {"a": 3, "c": 4} + assert dict(attrs) == expected diff --git a/tests/v3/test_buffer.py b/tests/v3/test_buffer.py index 5a313dc1ab..cde3f85780 100644 --- a/tests/v3/test_buffer.py +++ b/tests/v3/test_buffer.py @@ -32,6 +32,10 @@ cp = None +if TYPE_CHECKING: + import types + + def test_nd_array_like(xp: types.ModuleType) -> None: ary = xp.arange(10) assert isinstance(ary, ArrayLike) diff --git a/tests/v3/test_group.py b/tests/v3/test_group.py index 6e75294b7c..c8310f33e5 100644 --- a/tests/v3/test_group.py +++ b/tests/v3/test_group.py @@ -51,7 +51,7 @@ def test_group_init(store: Store, zarr_format: ZarrFormat) -> None: """ Test that initializing a group from an asyncgroup works. """ - agroup = sync(AsyncGroup.create(store=store, zarr_format=zarr_format)) + agroup = sync(AsyncGroup.from_store(store=store, zarr_format=zarr_format)) group = Group(agroup) assert group._async_group == agroup @@ -60,7 +60,7 @@ def test_group_name_properties(store: Store, zarr_format: ZarrFormat) -> None: """ Test basic properties of groups """ - root = Group.create(store=store, zarr_format=zarr_format) + root = Group.from_store(store=store, zarr_format=zarr_format) assert root.path == "" assert root.name == "/" assert root.basename == "" @@ -178,16 +178,18 @@ def test_group(store: Store, zarr_format: ZarrFormat) -> None: def test_group_create(store: Store, exists_ok: bool, zarr_format: ZarrFormat) -> None: """ - Test that `Group.create` works as expected. + Test that `Group.from_store` works as expected. """ attributes = {"foo": 100} - group = Group.create(store, attributes=attributes, zarr_format=zarr_format, exists_ok=exists_ok) + group = Group.from_store( + store, attributes=attributes, zarr_format=zarr_format, exists_ok=exists_ok + ) assert group.attrs == attributes if not exists_ok: with pytest.raises(ContainsGroupError): - group = Group.create( + group = Group.from_store( store, attributes=attributes, exists_ok=exists_ok, zarr_format=zarr_format ) @@ -203,7 +205,7 @@ def test_group_open(store: Store, zarr_format: ZarrFormat, exists_ok: bool) -> N # create the group attrs = {"path": "foo"} - group_created = Group.create( + group_created = Group.from_store( store, attributes=attrs, zarr_format=zarr_format, exists_ok=exists_ok ) assert group_created.attrs == attrs @@ -214,9 +216,9 @@ def test_group_open(store: Store, zarr_format: ZarrFormat, exists_ok: bool) -> N new_attrs = {"path": "bar"} if not exists_ok: with pytest.raises(ContainsGroupError): - Group.create(store, attributes=attrs, zarr_format=zarr_format, exists_ok=exists_ok) + Group.from_store(store, attributes=attrs, zarr_format=zarr_format, exists_ok=exists_ok) else: - group_created_again = Group.create( + group_created_again = Group.from_store( store, attributes=new_attrs, zarr_format=zarr_format, exists_ok=exists_ok ) assert group_created_again.attrs == new_attrs @@ -229,7 +231,7 @@ def test_group_getitem(store: Store, zarr_format: ZarrFormat) -> None: Test the `Group.__getitem__` method. """ - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") subarray = group.create_array(name="subarray", shape=(10,), chunk_shape=(10,)) @@ -246,7 +248,7 @@ def test_group_delitem(store: Store, zarr_format: ZarrFormat) -> None: if not store.supports_deletes: pytest.skip("store does not support deletes") - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) subgroup = group.create_group(name="subgroup") subarray = group.create_array(name="subarray", shape=(10,), chunk_shape=(10,)) @@ -267,7 +269,7 @@ def test_group_iter(store: Store, zarr_format: ZarrFormat) -> None: Test the `Group.__iter__` method. """ - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) with pytest.raises(NotImplementedError): [x for x in group] # type: ignore @@ -277,7 +279,7 @@ def test_group_len(store: Store, zarr_format: ZarrFormat) -> None: Test the `Group.__len__` method. """ - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) with pytest.raises(NotImplementedError): len(group) # type: ignore @@ -286,7 +288,7 @@ def test_group_setitem(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__setitem__` method. """ - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) with pytest.raises(NotImplementedError): group["key"] = 10 @@ -295,14 +297,14 @@ def test_group_contains(store: Store, zarr_format: ZarrFormat) -> None: """ Test the `Group.__contains__` method """ - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) assert "foo" not in group _ = group.create_group(name="foo") assert "foo" in group def test_group_child_iterators(store: Store, zarr_format: ZarrFormat): - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) expected_group_keys = ["g0", "g1"] expected_group_values = [group.create_group(name=name) for name in expected_group_keys] expected_groups = list(zip(expected_group_keys, expected_group_values, strict=False)) @@ -330,7 +332,7 @@ def test_group_update_attributes(store: Store, zarr_format: ZarrFormat) -> None: Test the behavior of `Group.update_attributes` """ attrs = {"foo": 100} - group = Group.create(store, zarr_format=zarr_format, attributes=attrs) + group = Group.from_store(store, zarr_format=zarr_format, attributes=attrs) assert group.attrs == attrs new_attrs = {"bar": 100} new_group = group.update_attributes(new_attrs) @@ -342,7 +344,7 @@ async def test_group_update_attributes_async(store: Store, zarr_format: ZarrForm Test the behavior of `Group.update_attributes_async` """ attrs = {"foo": 100} - group = Group.create(store, zarr_format=zarr_format, attributes=attrs) + group = Group.from_store(store, zarr_format=zarr_format, attributes=attrs) assert group.attrs == attrs new_attrs = {"bar": 100} new_group = await group.update_attributes_async(new_attrs) @@ -357,9 +359,9 @@ def test_group_create_array( method: Literal["create_array", "array"], ) -> None: """ - Test `Group.create_array` + Test `Group.from_store` """ - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) shape = (10, 10) dtype = "uint8" data = np.arange(np.prod(shape)).reshape(shape).astype(dtype) @@ -388,7 +390,7 @@ def test_group_array_creation( store: Store, zarr_format: ZarrFormat, ): - group = Group.create(store, zarr_format=zarr_format) + group = Group.from_store(store, zarr_format=zarr_format) shape = (10, 10) empty_array = group.empty(name="empty", shape=shape) assert isinstance(empty_array, Array) @@ -465,7 +467,7 @@ def test_group_creation_existing_node( Check that an existing array or group is handled as expected during group creation. """ spath = StorePath(store) - group = Group.create(spath, zarr_format=zarr_format) + group = Group.from_store(spath, zarr_format=zarr_format) expected_exception: type[ContainsArrayError] | type[ContainsGroupError] attributes: dict[str, JSON] = {"old": True} @@ -481,7 +483,7 @@ def test_group_creation_existing_node( new_attributes = {"new": True} if exists_ok: - node_new = Group.create( + node_new = Group.from_store( spath / "extant", attributes=new_attributes, zarr_format=zarr_format, @@ -490,7 +492,7 @@ def test_group_creation_existing_node( assert node_new.attrs == new_attributes else: with pytest.raises(expected_exception): - node_new = Group.create( + node_new = Group.from_store( spath / "extant", attributes=new_attributes, zarr_format=zarr_format, @@ -504,11 +506,11 @@ async def test_asyncgroup_create( zarr_format: ZarrFormat, ) -> None: """ - Test that `AsyncGroup.create` works as expected. + Test that `AsyncGroup.from_store` works as expected. """ spath = StorePath(store=store) attributes = {"foo": 100} - agroup = await AsyncGroup.create( + agroup = await AsyncGroup.from_store( store, attributes=attributes, exists_ok=exists_ok, @@ -520,7 +522,7 @@ async def test_asyncgroup_create( if not exists_ok: with pytest.raises(ContainsGroupError): - agroup = await AsyncGroup.create( + agroup = await AsyncGroup.from_store( spath, attributes=attributes, exists_ok=exists_ok, @@ -532,7 +534,7 @@ async def test_asyncgroup_create( spath / collision_name, shape=(10,), dtype="uint8", zarr_format=zarr_format ) with pytest.raises(ContainsArrayError): - _ = await AsyncGroup.create( + _ = await AsyncGroup.from_store( StorePath(store=store) / collision_name, attributes=attributes, exists_ok=exists_ok, @@ -542,13 +544,13 @@ async def test_asyncgroup_create( async def test_asyncgroup_attrs(store: Store, zarr_format: ZarrFormat) -> None: attributes = {"foo": 100} - agroup = await AsyncGroup.create(store, zarr_format=zarr_format, attributes=attributes) + agroup = await AsyncGroup.from_store(store, zarr_format=zarr_format, attributes=attributes) assert agroup.attrs == agroup.metadata.attributes == attributes async def test_asyncgroup_info(store: Store, zarr_format: ZarrFormat) -> None: - agroup = await AsyncGroup.create( # noqa + agroup = await AsyncGroup.from_store( # noqa store, zarr_format=zarr_format, ) @@ -564,7 +566,7 @@ async def test_asyncgroup_open( Create an `AsyncGroup`, then ensure that we can open it using `AsyncGroup.open` """ attributes = {"foo": 100} - group_w = await AsyncGroup.create( + group_w = await AsyncGroup.from_store( store=store, attributes=attributes, exists_ok=False, @@ -581,7 +583,7 @@ async def test_asyncgroup_open_wrong_format( store: Store, zarr_format: ZarrFormat, ) -> None: - _ = await AsyncGroup.create(store=store, exists_ok=False, zarr_format=zarr_format) + _ = await AsyncGroup.from_store(store=store, exists_ok=False, zarr_format=zarr_format) zarr_format_wrong: ZarrFormat # try opening with the wrong zarr format if zarr_format == 3: @@ -624,7 +626,7 @@ async def test_asyncgroup_getitem(store: Store, zarr_format: ZarrFormat) -> None Create an `AsyncGroup`, then create members of that group, and ensure that we can access those members via the `AsyncGroup.getitem` method. """ - agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) array_name = "sub_array" sub_array = await agroup.create_array( @@ -645,7 +647,7 @@ async def test_asyncgroup_delitem(store: Store, zarr_format: ZarrFormat) -> None if not store.supports_deletes: pytest.skip("store does not support deletes") - agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) array_name = "sub_array" _ = await agroup.create_array( name=array_name, shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} @@ -677,7 +679,7 @@ async def test_asyncgroup_create_group( store: Store, zarr_format: ZarrFormat, ) -> None: - agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) sub_node_path = "sub_group" attributes = {"foo": 999} subnode = await agroup.create_group(name=sub_node_path, attributes=attributes) @@ -697,11 +699,11 @@ async def test_asyncgroup_create_array( specified in create_array are present on the resulting array. """ - agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) if not exists_ok: with pytest.raises(ContainsGroupError): - agroup = await AsyncGroup.create(store=store, zarr_format=zarr_format) + agroup = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) shape = (10,) dtype = "uint8" @@ -734,7 +736,7 @@ async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrForma """ attributes_old = {"foo": 10} attributes_new = {"baz": "new"} - agroup = await AsyncGroup.create( + agroup = await AsyncGroup.from_store( store=store, zarr_format=zarr_format, attributes=attributes_old ) @@ -745,7 +747,7 @@ async def test_asyncgroup_update_attributes(store: Store, zarr_format: ZarrForma @pytest.mark.parametrize("store", ("local",), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) async def test_serializable_async_group(store: LocalStore, zarr_format: ZarrFormat) -> None: - expected = await AsyncGroup.create( + expected = await AsyncGroup.from_store( store=store, attributes={"foo": 999}, zarr_format=zarr_format ) p = pickle.dumps(expected) @@ -756,7 +758,7 @@ async def test_serializable_async_group(store: LocalStore, zarr_format: ZarrForm @pytest.mark.parametrize("store", ("local",), indirect=["store"]) @pytest.mark.parametrize("zarr_format", (2, 3)) def test_serializable_sync_group(store: LocalStore, zarr_format: ZarrFormat) -> None: - expected = Group.create(store=store, attributes={"foo": 999}, zarr_format=zarr_format) + expected = Group.from_store(store=store, attributes={"foo": 999}, zarr_format=zarr_format) p = pickle.dumps(expected) actual = pickle.loads(p) @@ -817,7 +819,7 @@ async def test_group_members_async(store: LocalStore | MemoryStore) -> None: async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) # create foo group _ = await root.create_group("foo", attributes={"foo": 100}) @@ -845,7 +847,7 @@ async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrF async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) # create foo group _ = await root.create_group("foo", attributes={"foo": 100}) # create bar group @@ -866,7 +868,7 @@ async def test_require_groups(store: LocalStore | MemoryStore, zarr_format: Zarr async def test_create_dataset(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) with pytest.warns(DeprecationWarning): foo = await root.create_dataset("foo", shape=(10,), dtype="uint8") assert foo.shape == (10,) @@ -880,7 +882,7 @@ async def test_create_dataset(store: LocalStore | MemoryStore, zarr_format: Zarr async def test_require_array(store: LocalStore | MemoryStore, zarr_format: ZarrFormat) -> None: - root = await AsyncGroup.create(store=store, zarr_format=zarr_format) + root = await AsyncGroup.from_store(store=store, zarr_format=zarr_format) foo1 = await root.require_array("foo", shape=(10,), dtype="i8", attributes={"foo": 101}) assert foo1.attrs == {"foo": 101} foo2 = await root.require_array("foo", shape=(10,), dtype="i8") diff --git a/tests/v3/test_sync.py b/tests/v3/test_sync.py index 22834747e7..864c9e01cb 100644 --- a/tests/v3/test_sync.py +++ b/tests/v3/test_sync.py @@ -4,7 +4,9 @@ import pytest +import zarr from zarr.core.sync import SyncError, SyncMixin, _get_lock, _get_loop, sync +from zarr.store.memory import MemoryStore @pytest.fixture(params=[True, False]) @@ -121,3 +123,9 @@ def bar(self) -> list[int]: foo = SyncFoo(async_foo) assert foo.foo() == "foo" assert foo.bar() == list(range(10)) + + +def test_open_positional_args_deprecate(): + store = MemoryStore({}, mode="w") + with pytest.warns(FutureWarning, match="pass"): + zarr.open(store, "w", shape=(1,)) From fb28fa50cf89dc1519d3150d56a702f322a1931c Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Mon, 23 Sep 2024 14:48:00 -0600 Subject: [PATCH 0170/1043] fix typo in store integration test (#2223) --- src/zarr/testing/store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index 8a9f27e4b8..ebd4b85c90 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -58,7 +58,7 @@ def test_store_eq(self, store: S, store_kwargs: dict[str, Any]) -> None: store2 = self.store_cls(**store_kwargs) assert store == store2 - def test_serizalizable_store(self, store: S) -> None: + def test_serializable_store(self, store: S) -> None: foo = pickle.dumps(store) assert pickle.loads(foo) == store From 2d3a36c7cc71f362be7f4b2cd128da727f24810a Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 23 Sep 2024 13:52:48 -0700 Subject: [PATCH 0171/1043] fix: validate v3 dtypes when loading/creating v3 metadata (#2209) --- pyproject.toml | 1 + src/zarr/core/array_spec.py | 5 +- src/zarr/core/common.py | 7 -- src/zarr/core/metadata/v2.py | 29 ++++- src/zarr/core/metadata/v3.py | 189 +++++++++++++++++++++++------- src/zarr/testing/strategies.py | 9 +- tests/v3/test_api.py | 4 +- tests/v3/test_array.py | 6 +- tests/v3/test_metadata/test_v3.py | 121 ++++++++++++++++--- 9 files changed, 291 insertions(+), 80 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41c2006b2f..63a58ac795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -274,6 +274,7 @@ filterwarnings = [ "ignore:PY_SSIZE_T_CLEAN will be required.*:DeprecationWarning", "ignore:The loop argument is deprecated since Python 3.8.*:DeprecationWarning", "ignore:Creating a zarr.buffer.gpu.*:UserWarning", + "ignore:Duplicate name:UserWarning", # from ZipFile ] markers = [ "gpu: mark a test as requiring CuPy and GPU" diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py index e64a962bc3..1a251a0a4b 100644 --- a/src/zarr/core/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal -from zarr.core.common import parse_dtype, parse_fill_value, parse_order, parse_shapelike +from zarr.core.common import parse_fill_value, parse_order, parse_shapelike if TYPE_CHECKING: import numpy as np @@ -29,12 +29,11 @@ def __init__( prototype: BufferPrototype, ) -> None: shape_parsed = parse_shapelike(shape) - dtype_parsed = parse_dtype(dtype) fill_value_parsed = parse_fill_value(fill_value) order_parsed = parse_order(order) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "dtype", dtype_parsed) + object.__setattr__(self, "dtype", dtype) object.__setattr__(self, "fill_value", fill_value_parsed) object.__setattr__(self, "order", order_parsed) object.__setattr__(self, "prototype", prototype) diff --git a/src/zarr/core/common.py b/src/zarr/core/common.py index 8ebe5160bd..6847bd419f 100644 --- a/src/zarr/core/common.py +++ b/src/zarr/core/common.py @@ -19,8 +19,6 @@ if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator -import numpy as np -import numpy.typing as npt ZARR_JSON = "zarr.json" ZARRAY_JSON = ".zarray" @@ -155,11 +153,6 @@ def parse_shapelike(data: int | Iterable[int]) -> tuple[int, ...]: return data_tuple -def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]: - # todo: real validation - return np.dtype(data) - - def parse_fill_value(data: Any) -> Any: # todo: real validation return data diff --git a/src/zarr/core/metadata/v2.py b/src/zarr/core/metadata/v2.py index af7821bea7..34bdbb537f 100644 --- a/src/zarr/core/metadata/v2.py +++ b/src/zarr/core/metadata/v2.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Iterable +from enum import Enum from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -21,7 +22,7 @@ from zarr.core.array_spec import ArraySpec from zarr.core.chunk_grids import RegularChunkGrid from zarr.core.chunk_key_encodings import parse_separator -from zarr.core.common import ZARRAY_JSON, ZATTRS_JSON, parse_dtype, parse_shapelike +from zarr.core.common import ZARRAY_JSON, ZATTRS_JSON, parse_shapelike from zarr.core.config import config, parse_indexing_order from zarr.core.metadata.common import ArrayMetadata, parse_attributes @@ -100,9 +101,24 @@ def _json_convert( else: return o.descr if np.isscalar(o): - # convert numpy scalar to python type, and pass - # python types through - return getattr(o, "item", lambda: o)() + out: Any + if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): + # https://github.com/zarr-developers/zarr-python/issues/2119 + # `.item()` on a datetime type might or might not return an + # integer, depending on the value. + # Explicitly cast to an int first, and then grab .item() + out = o.view("i8").item() + else: + # convert numpy scalar to python type, and pass + # python types through + out = getattr(o, "item", lambda: o)() + if isinstance(out, complex): + # python complex types are not JSON serializable, so we use the + # serialization defined in the zarr v3 spec + return [out.real, out.imag] + return out + if isinstance(o, Enum): + return o.name raise TypeError zarray_dict = self.to_dict() @@ -157,6 +173,11 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: return replace(self, attributes=attributes) +def parse_dtype(data: npt.DTypeLike) -> np.dtype[Any]: + # todo: real validation + return np.dtype(data) + + def parse_zarr_format(data: object) -> Literal[2]: if data == 2: return 2 diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 10047cbb93..603cd343af 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, cast, overload if TYPE_CHECKING: @@ -24,7 +25,7 @@ from zarr.core.buffer import default_buffer_prototype from zarr.core.chunk_grids import ChunkGrid, RegularChunkGrid from zarr.core.chunk_key_encodings import ChunkKeyEncoding -from zarr.core.common import ZARR_JSON, parse_dtype, parse_named_configuration, parse_shapelike +from zarr.core.common import ZARR_JSON, parse_named_configuration, parse_shapelike from zarr.core.config import config from zarr.core.metadata.common import ArrayMetadata, parse_attributes from zarr.registry import get_codec_class @@ -70,6 +71,66 @@ def parse_dimension_names(data: object) -> tuple[str | None, ...] | None: raise TypeError(msg) +class V3JsonEncoder(json.JSONEncoder): + def __init__(self, *args: Any, **kwargs: Any): + self.indent = kwargs.pop("indent", config.get("json_indent")) + super().__init__(*args, **kwargs) + + def default(self, o: object) -> Any: + if isinstance(o, np.dtype): + return str(o) + if np.isscalar(o): + out: Any + if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): + # https://github.com/zarr-developers/zarr-python/issues/2119 + # `.item()` on a datetime type might or might not return an + # integer, depending on the value. + # Explicitly cast to an int first, and then grab .item() + out = o.view("i8").item() + else: + # convert numpy scalar to python type, and pass + # python types through + out = getattr(o, "item", lambda: o)() + if isinstance(out, complex): + # python complex types are not JSON serializable, so we use the + # serialization defined in the zarr v3 spec + return [out.real, out.imag] + elif np.isnan(out): + return "NaN" + elif np.isinf(out): + return "Infinity" if out > 0 else "-Infinity" + return out + elif isinstance(o, Enum): + return o.name + # this serializes numcodecs compressors + # todo: implement to_dict for codecs + elif isinstance(o, numcodecs.abc.Codec): + config: dict[str, Any] = o.get_config() + return config + else: + return super().default(o) + + +def _replace_special_floats(obj: object) -> Any: + """Helper function to replace NaN/Inf/-Inf values with special strings + + Note: this cannot be done in the V3JsonEncoder because Python's `json.dumps` optimistically + converts NaN/Inf values to special types outside of the encoding step. + """ + if isinstance(obj, float): + if np.isnan(obj): + return "NaN" + elif np.isinf(obj): + return "Infinity" if obj > 0 else "-Infinity" + elif isinstance(obj, dict): + # Recursively replace in dictionaries + return {k: _replace_special_floats(v) for k, v in obj.items()} + elif isinstance(obj, list): + # Recursively replace in lists + return [_replace_special_floats(item) for item in obj] + return obj + + @dataclass(frozen=True, kw_only=True) class ArrayV3Metadata(ArrayMetadata): shape: ChunkCoords @@ -169,41 +230,8 @@ def encode_chunk_key(self, chunk_coords: ChunkCoords) -> str: return self.chunk_key_encoding.encode_chunk_key(chunk_coords) def to_buffer_dict(self, prototype: BufferPrototype) -> dict[str, Buffer]: - def _json_convert(o: object) -> Any: - if isinstance(o, np.dtype): - return str(o) - if np.isscalar(o): - out: Any - if hasattr(o, "dtype") and o.dtype.kind == "M" and hasattr(o, "view"): - # https://github.com/zarr-developers/zarr-python/issues/2119 - # `.item()` on a datetime type might or might not return an - # integer, depending on the value. - # Explicitly cast to an int first, and then grab .item() - out = o.view("i8").item() - else: - # convert numpy scalar to python type, and pass - # python types through - out = getattr(o, "item", lambda: o)() - if isinstance(out, complex): - # python complex types are not JSON serializable, so we use the - # serialization defined in the zarr v3 spec - return [out.real, out.imag] - return out - if isinstance(o, Enum): - return o.name - # this serializes numcodecs compressors - # todo: implement to_dict for codecs - elif isinstance(o, numcodecs.abc.Codec): - config: dict[str, Any] = o.get_config() - return config - raise TypeError - - json_indent = config.get("json_indent") - return { - ZARR_JSON: prototype.buffer.from_bytes( - json.dumps(self.to_dict(), default=_json_convert, indent=json_indent).encode() - ) - } + d = _replace_special_floats(self.to_dict()) + return {ZARR_JSON: prototype.buffer.from_bytes(json.dumps(d, cls=V3JsonEncoder).encode())} @classmethod def from_dict(cls, data: dict[str, JSON]) -> Self: @@ -215,6 +243,9 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: # check that the node_type attribute is correct _ = parse_node_type_array(_data.pop("node_type")) + # check that the data_type attribute is valid + _ = DataType(_data["data_type"]) + # dimension_names key is optional, normalize missing to `None` _data["dimension_names"] = _data.pop("dimension_names", None) # attributes key is optional, normalize missing to `None` @@ -260,23 +291,38 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: @overload -def parse_fill_value(fill_value: object, dtype: BOOL_DTYPE) -> BOOL: ... +def parse_fill_value( + fill_value: int | float | complex | str | bytes | np.generic | Sequence[Any] | bool | None, + dtype: BOOL_DTYPE, +) -> BOOL: ... @overload -def parse_fill_value(fill_value: object, dtype: INTEGER_DTYPE) -> INTEGER: ... +def parse_fill_value( + fill_value: int | float | complex | str | bytes | np.generic | Sequence[Any] | bool | None, + dtype: INTEGER_DTYPE, +) -> INTEGER: ... @overload -def parse_fill_value(fill_value: object, dtype: FLOAT_DTYPE) -> FLOAT: ... +def parse_fill_value( + fill_value: int | float | complex | str | bytes | np.generic | Sequence[Any] | bool | None, + dtype: FLOAT_DTYPE, +) -> FLOAT: ... @overload -def parse_fill_value(fill_value: object, dtype: COMPLEX_DTYPE) -> COMPLEX: ... +def parse_fill_value( + fill_value: int | float | complex | str | bytes | np.generic | Sequence[Any] | bool | None, + dtype: COMPLEX_DTYPE, +) -> COMPLEX: ... @overload -def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: +def parse_fill_value( + fill_value: int | float | complex | str | bytes | np.generic | Sequence[Any] | bool | None, + dtype: np.dtype[Any], +) -> Any: # This dtype[Any] is unfortunately necessary right now. # See https://github.com/zarr-developers/zarr-python/issues/2131#issuecomment-2318010899 # for more details, but `dtype` here (which comes from `parse_dtype`) @@ -288,7 +334,7 @@ def parse_fill_value(fill_value: object, dtype: np.dtype[Any]) -> Any: def parse_fill_value( - fill_value: object, + fill_value: int | float | complex | str | bytes | np.generic | Sequence[Any] | bool | None, dtype: BOOL_DTYPE | INTEGER_DTYPE | FLOAT_DTYPE | COMPLEX_DTYPE | np.dtype[Any], ) -> BOOL | INTEGER | FLOAT | COMPLEX | Any: """ @@ -322,13 +368,40 @@ def parse_fill_value( else: msg = ( f"Got an invalid fill value for complex data type {dtype}." - f"Expected a sequence with 2 elements, but {fill_value} has " + f"Expected a sequence with 2 elements, but {fill_value!r} has " f"length {len(fill_value)}." ) raise ValueError(msg) - msg = f"Cannot parse non-string sequence {fill_value} as a scalar with type {dtype}." + msg = f"Cannot parse non-string sequence {fill_value!r} as a scalar with type {dtype}." raise TypeError(msg) - return dtype.type(fill_value) # type: ignore[arg-type] + + # Cast the fill_value to the given dtype + try: + # This warning filter can be removed after Zarr supports numpy>=2.0 + # The warning is saying that the future behavior of out of bounds casting will be to raise + # an OverflowError. In the meantime, we allow overflow and catch cases where + # fill_value != casted_value below. + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + casted_value = np.dtype(dtype).type(fill_value) + except (ValueError, OverflowError, TypeError) as e: + raise ValueError(f"fill value {fill_value!r} is not valid for dtype {dtype}") from e + # Check if the value is still representable by the dtype + if fill_value == "NaN" and np.isnan(casted_value): + pass + elif fill_value in ["Infinity", "-Infinity"] and not np.isfinite(casted_value): + pass + elif dtype.kind == "f": + # float comparison is not exact, especially when dtype int: @@ -360,8 +436,11 @@ def byte_count(self) -> int: DataType.uint16: 2, DataType.uint32: 4, DataType.uint64: 8, + DataType.float16: 2, DataType.float32: 4, DataType.float64: 8, + DataType.complex64: 8, + DataType.complex128: 16, } return data_type_byte_counts[self] @@ -381,8 +460,11 @@ def to_numpy_shortname(self) -> str: DataType.uint16: "u2", DataType.uint32: "u4", DataType.uint64: "u8", + DataType.float16: "f2", DataType.float32: "f4", DataType.float64: "f8", + DataType.complex64: "c8", + DataType.complex128: "c16", } return data_type_to_numpy[self] @@ -399,7 +481,24 @@ def from_dtype(cls, dtype: np.dtype[Any]) -> DataType: " np.dtype[Any]: + try: + dtype = np.dtype(data) + except (ValueError, TypeError) as e: + raise ValueError(f"Invalid V3 data_type: {data}") from e + # check that this is a valid v3 data_type + try: + _ = DataType.from_dtype(dtype) + except KeyError as e: + raise ValueError(f"Invalid V3 data_type: {dtype}") from e + + return dtype diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py index 2f4ca8a697..3f9d1264d9 100644 --- a/src/zarr/testing/strategies.py +++ b/src/zarr/testing/strategies.py @@ -35,11 +35,14 @@ paths = st.lists(node_names, min_size=1).map(lambda x: "/".join(x)) | st.just("/") np_arrays = npst.arrays( # TODO: re-enable timedeltas once they are supported - dtype=npst.scalar_dtypes().filter(lambda x: x.kind != "m"), + dtype=npst.scalar_dtypes().filter( + lambda x: (x.kind not in ["m", "M"]) and (x.byteorder not in [">"]) + ), shape=npst.array_shapes(max_dims=4), ) stores = st.builds(MemoryStore, st.just({}), mode=st.just("w")) compressors = st.sampled_from([None, "default"]) +format = st.sampled_from([2, 3]) @st.composite # type: ignore[misc] @@ -69,12 +72,14 @@ def arrays( paths: st.SearchStrategy[None | str] = paths, array_names: st.SearchStrategy = array_names, attrs: st.SearchStrategy = attrs, + format: st.SearchStrategy = format, ) -> Array: store = draw(stores) nparray, chunks = draw(np_array_and_chunks(arrays=arrays)) path = draw(paths) name = draw(array_names) attributes = draw(attrs) + zarr_format = draw(format) # compressor = draw(compressors) # TODO: clean this up @@ -99,7 +104,7 @@ def arrays( expected_attrs = {} if attributes is None else attributes array_path = path + ("/" if not path.endswith("/") else "") + name - root = Group.from_store(store) + root = Group.from_store(store, zarr_format=zarr_format) fill_value_args: tuple[Any, ...] = tuple() if nparray.dtype.kind == "M": m = re.search(r"\[(.+)\]", nparray.dtype.str) diff --git a/tests/v3/test_api.py b/tests/v3/test_api.py index 728638c871..1b4330eef3 100644 --- a/tests/v3/test_api.py +++ b/tests/v3/test_api.py @@ -119,9 +119,11 @@ def test_open_with_mode_r(tmp_path: pathlib.Path) -> None: # 'r' means read only (must exist) with pytest.raises(FileNotFoundError): zarr.open(store=tmp_path, mode="r") - zarr.ones(store=tmp_path, shape=(3, 3)) + z1 = zarr.ones(store=tmp_path, shape=(3, 3)) + assert z1.fill_value == 1 z2 = zarr.open(store=tmp_path, mode="r") assert isinstance(z2, Array) + assert z2.fill_value == 1 assert (z2[:] == 1).all() with pytest.raises(ValueError): z2[:] = 3 diff --git a/tests/v3/test_array.py b/tests/v3/test_array.py index fe5c782a1b..b3362c52b0 100644 --- a/tests/v3/test_array.py +++ b/tests/v3/test_array.py @@ -122,8 +122,10 @@ def test_array_v3_fill_value_default( @pytest.mark.parametrize("store", ["memory"], indirect=True) -@pytest.mark.parametrize("fill_value", [False, 0.0, 1, 2.3]) -@pytest.mark.parametrize("dtype_str", ["bool", "uint8", "float32", "complex64"]) +@pytest.mark.parametrize( + "dtype_str,fill_value", + [("bool", True), ("uint8", 99), ("float32", -99.9), ("complex64", 3 + 4j)], +) def test_array_v3_fill_value(store: MemoryStore, fill_value: int, dtype_str: str) -> None: shape = (10,) arr = Array.create( diff --git a/tests/v3/test_metadata/test_v3.py b/tests/v3/test_metadata/test_v3.py index 0a545dfb9d..d4cf0c73e3 100644 --- a/tests/v3/test_metadata/test_v3.py +++ b/tests/v3/test_metadata/test_v3.py @@ -19,7 +19,12 @@ import numpy as np import pytest -from zarr.core.metadata.v3 import parse_dimension_names, parse_fill_value, parse_zarr_format +from zarr.core.metadata.v3 import ( + parse_dimension_names, + parse_dtype, + parse_fill_value, + parse_zarr_format, +) bool_dtypes = ("bool",) @@ -76,14 +81,34 @@ def test_parse_auto_fill_value(dtype_str: str) -> None: assert parse_fill_value(fill_value, dtype) == dtype.type(0) -@pytest.mark.parametrize("fill_value", [0, 1.11, False, True]) -@pytest.mark.parametrize("dtype_str", dtypes) +@pytest.mark.parametrize( + "fill_value,dtype_str", + [ + (True, "bool"), + (False, "bool"), + (-8, "int8"), + (0, "int16"), + (1e10, "uint64"), + (-999, "float32"), + (1e32, "float64"), + (float("NaN"), "float64"), + (np.nan, "float64"), + (np.inf, "float64"), + (-1 * np.inf, "float64"), + (0j, "complex64"), + ], +) def test_parse_fill_value_valid(fill_value: Any, dtype_str: str) -> None: """ Test that parse_fill_value(fill_value, dtype) casts fill_value to the given dtype. """ dtype = np.dtype(dtype_str) - assert parse_fill_value(fill_value, dtype) == dtype.type(fill_value) + parsed = parse_fill_value(fill_value, dtype) + + if np.isnan(fill_value): + assert np.isnan(parsed) + else: + assert parsed == dtype.type(fill_value) @pytest.mark.parametrize("fill_value", ["not a valid value"]) @@ -138,8 +163,7 @@ def test_parse_fill_value_invalid_type(fill_value: Any, dtype_str: str) -> None: This test excludes bool because the bool constructor takes anything. """ dtype = np.dtype(dtype_str) - match = "must be" - with pytest.raises(TypeError, match=match): + with pytest.raises(ValueError, match=r"fill value .* is not valid for dtype .*"): parse_fill_value(fill_value, dtype) @@ -234,22 +258,87 @@ def test_metadata_to_dict( assert observed == expected -@pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) -@pytest.mark.parametrize("precision", ["ns", "D"]) -async def test_datetime_metadata(fill_value: int, precision: str) -> None: +# @pytest.mark.parametrize("fill_value", [-1, 0, 1, 2932897]) +# @pytest.mark.parametrize("precision", ["ns", "D"]) +# async def test_datetime_metadata(fill_value: int, precision: str) -> None: +# metadata_dict = { +# "zarr_format": 3, +# "node_type": "array", +# "shape": (1,), +# "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, +# "data_type": f" None: metadata_dict = { "zarr_format": 3, "node_type": "array", "shape": (1,), "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, - "data_type": f" None: + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": data_type, + "chunk_key_encoding": {"name": "default", "separator": "."}, + "codecs": (), + "fill_value": fill_value, # this is not a valid fill value for uint8 + } + with pytest.raises(ValueError, match=r"fill value .* is not valid for dtype .*"): + ArrayV3Metadata.from_dict(metadata_dict) + + +@pytest.mark.parametrize("fill_value", [("NaN"), "Infinity", "-Infinity"]) +async def test_special_float_fill_values(fill_value: str) -> None: + metadata_dict = { + "zarr_format": 3, + "node_type": "array", + "shape": (1,), + "chunk_grid": {"name": "regular", "configuration": {"chunk_shape": (1,)}}, + "data_type": "float64", + "chunk_key_encoding": {"name": "default", "separator": "."}, + "codecs": (), + "fill_value": fill_value, # this is not a valid fill value for uint8 + } + m = ArrayV3Metadata.from_dict(metadata_dict) + d = json.loads(m.to_buffer_dict(default_buffer_prototype())["zarr.json"].to_bytes()) + assert m.fill_value is not None + if fill_value == "NaN": + assert np.isnan(m.fill_value) + assert d["fill_value"] == "NaN" + elif fill_value == "Infinity": + assert np.isposinf(m.fill_value) + assert d["fill_value"] == "Infinity" + elif fill_value == "-Infinity": + assert np.isneginf(m.fill_value) + assert d["fill_value"] == "-Infinity" From cd7321b6481e1d64ea21c97de8cd61685804d476 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:59:42 -0700 Subject: [PATCH 0172/1043] chore: update pre-commit hooks (#2222) --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 79c68418da..99a69dc541 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.5 + rev: v0.6.7 hooks: - id: ruff args: ["--fix", "--show-fixes"] From 2a78497f5307fa43db177973beb91ee8c0dd1b0b Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 23 Sep 2024 22:46:13 -0500 Subject: [PATCH 0173/1043] Added Attributes.asdict (#2221) --- src/zarr/core/attributes.py | 3 +++ tests/v3/test_attributes.py | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/src/zarr/core/attributes.py b/src/zarr/core/attributes.py index 62ff5fc935..913a4b74ed 100644 --- a/src/zarr/core/attributes.py +++ b/src/zarr/core/attributes.py @@ -51,3 +51,6 @@ def put(self, d: dict[str, JSON]) -> None: {'a': 3, 'c': 4} """ self._obj = self._obj.update_attributes(d) + + def asdict(self) -> dict[str, JSON]: + return dict(self._obj.metadata.attributes) diff --git a/tests/v3/test_attributes.py b/tests/v3/test_attributes.py index 65b6a02e8d..14c60492be 100644 --- a/tests/v3/test_attributes.py +++ b/tests/v3/test_attributes.py @@ -11,3 +11,12 @@ def test_put() -> None: attrs.put({"a": 3, "c": 4}) expected = {"a": 3, "c": 4} assert dict(attrs) == expected + + +def test_asdict() -> None: + store = zarr.store.MemoryStore({}, mode="w") + attrs = zarr.core.attributes.Attributes( + zarr.Group.from_store(store, attributes={"a": 1, "b": 2}) + ) + result = attrs.asdict() + assert result == {"a": 1, "b": 2} From ee2a3c60ab014803edba0a8b936f56e66b2c357b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:46:42 -0600 Subject: [PATCH 0174/1043] Bump pypa/gh-action-pypi-publish in the actions group (#2219) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.1 to 1.10.2 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.1...v1.10.2) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index bcff0f6154..b03ea06bf7 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -55,7 +55,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.1 + - uses: pypa/gh-action-pypi-publish@v1.10.2 with: user: __token__ password: ${{ secrets.pypi_password }} From a0110796ac246d2996e9e8bffcbf5a436b0d9bf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Sep 2024 21:47:16 -0600 Subject: [PATCH 0175/1043] Bump pypa/gh-action-pypi-publish in the actions group (#2220) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.1 to 1.10.2 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.1...v1.10.2) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-patch dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index 2d7cbd49b6..dccfbc208f 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -64,7 +64,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.1 + - uses: pypa/gh-action-pypi-publish@v1.10.2 with: user: __token__ password: ${{ secrets.pypi_password }} From a4b250979cc3028941abfba29ac85be6bfe65731 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 24 Sep 2024 08:13:53 -0700 Subject: [PATCH 0176/1043] Fix/empty listdir (#2225) * fix(store): report store is empty if no files are present in root directory * parse dtype again --- src/zarr/core/array_spec.py | 7 ++++--- src/zarr/store/local.py | 8 ++++++-- tests/v3/test_store/test_local.py | 5 +++++ 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/zarr/core/array_spec.py b/src/zarr/core/array_spec.py index 1a251a0a4b..e84a81cb05 100644 --- a/src/zarr/core/array_spec.py +++ b/src/zarr/core/array_spec.py @@ -3,11 +3,11 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal +import numpy as np + from zarr.core.common import parse_fill_value, parse_order, parse_shapelike if TYPE_CHECKING: - import numpy as np - from zarr.core.buffer import BufferPrototype from zarr.core.common import ChunkCoords @@ -29,11 +29,12 @@ def __init__( prototype: BufferPrototype, ) -> None: shape_parsed = parse_shapelike(shape) + dtype_parsed = np.dtype(dtype) fill_value_parsed = parse_fill_value(fill_value) order_parsed = parse_order(order) object.__setattr__(self, "shape", shape_parsed) - object.__setattr__(self, "dtype", dtype) + object.__setattr__(self, "dtype", dtype_parsed) object.__setattr__(self, "fill_value", fill_value_parsed) object.__setattr__(self, "order", order_parsed) object.__setattr__(self, "prototype", prototype) diff --git a/src/zarr/store/local.py b/src/zarr/store/local.py index c78837586f..fd209cd7c3 100644 --- a/src/zarr/store/local.py +++ b/src/zarr/store/local.py @@ -93,11 +93,15 @@ async def clear(self) -> None: async def empty(self) -> bool: try: - subpaths = os.listdir(self.root) + with os.scandir(self.root) as it: + for entry in it: + if entry.is_file(): + # stop once a file is found + return False except FileNotFoundError: return True else: - return not subpaths + return True def __str__(self) -> str: return f"file://{self.root}" diff --git a/tests/v3/test_store/test_local.py b/tests/v3/test_store/test_local.py index 5f1dde3fcc..bdd909c285 100644 --- a/tests/v3/test_store/test_local.py +++ b/tests/v3/test_store/test_local.py @@ -35,3 +35,8 @@ def test_store_supports_partial_writes(self, store: LocalStore) -> None: def test_store_supports_listing(self, store: LocalStore) -> None: assert store.supports_listing + + async def test_empty_with_empty_subdir(self, store: LocalStore) -> None: + assert await store.empty() + (store.root / "foo/bar").mkdir(parents=True) + assert await store.empty() From 30e2bc3ae0b7949cca9f477c5a43c6913631e3ef Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Tue, 24 Sep 2024 08:14:38 -0700 Subject: [PATCH 0177/1043] remove outdated v2 source code and tests from v3 branch (#2182) --- src/zarr/v2/__init__.py | 54 - src/zarr/v2/_storage/__init__.py | 0 src/zarr/v2/_storage/absstore.py | 224 -- src/zarr/v2/_storage/store.py | 226 -- src/zarr/v2/attrs.py | 158 - src/zarr/v2/codecs.py | 4 - src/zarr/v2/context.py | 19 - src/zarr/v2/convenience.py | 1284 -------- src/zarr/v2/core.py | 2855 ----------------- src/zarr/v2/creation.py | 707 ---- src/zarr/v2/errors.py | 80 - src/zarr/v2/hierarchy.py | 1401 -------- src/zarr/v2/indexing.py | 1074 ------- src/zarr/v2/meta.py | 302 -- src/zarr/v2/meta_v1.py | 64 - src/zarr/v2/n5.py | 897 ------ src/zarr/v2/storage.py | 2822 ---------------- src/zarr/v2/sync.py | 48 - src/zarr/v2/util.py | 788 ----- tests/v2/__init__.py | 0 tests/v2/conftest.py | 7 - tests/v2/fixture/.zgroup | 3 - .../fixture/dimension_separator/flat/.zarray | 23 - tests/v2/fixture/dimension_separator/flat/0.0 | Bin 48 -> 0 bytes .../dimension_separator/flat_legacy/.zarray | 22 - .../dimension_separator/flat_legacy/0.0 | Bin 48 -> 0 bytes .../dimension_separator/nested/.zarray | 23 - .../v2/fixture/dimension_separator/nested/0/0 | Bin 48 -> 0 bytes .../dimension_separator/nested_legacy/.zarray | 22 - .../dimension_separator/nested_legacy/0/0 | Bin 48 -> 0 bytes .../fixture/test_format_compatibility/.zgroup | 3 - .../test_format_compatibility/array_0/.zgroup | 3 - .../array_0/compressor_0/.zarray | 14 - .../array_0/compressor_0/0 | Bin 600 -> 0 bytes .../array_0/compressor_0/1 | Bin 600 -> 0 bytes .../array_0/compressor_1/.zarray | 17 - .../array_0/compressor_1/0 | Bin 284 -> 0 bytes .../array_0/compressor_1/1 | Bin 285 -> 0 bytes .../array_0/compressor_2/.zarray | 17 - .../array_0/compressor_2/0 | Bin 525 -> 0 bytes .../array_0/compressor_2/1 | Bin 500 -> 0 bytes .../array_0/compressor_3/.zarray | 20 - .../array_0/compressor_3/0 | Bin 300 -> 0 bytes .../array_0/compressor_3/1 | Bin 303 -> 0 bytes .../array_0/compressor_4/.zarray | 20 - .../array_0/compressor_4/0 | Bin 300 -> 0 bytes .../array_0/compressor_4/1 | Bin 303 -> 0 bytes .../array_0/compressor_5/.zarray | 20 - .../array_0/compressor_5/0 | Bin 121 -> 0 bytes .../array_0/compressor_5/1 | Bin 135 -> 0 bytes .../array_0/compressor_6/.zarray | 20 - .../array_0/compressor_6/0 | Bin 616 -> 0 bytes .../array_0/compressor_6/1 | Bin 556 -> 0 bytes .../test_format_compatibility/array_1/.zgroup | 3 - .../array_1/compressor_0/.zarray | 14 - .../array_1/compressor_0/0 | Bin 1200 -> 0 bytes .../array_1/compressor_0/1 | Bin 1200 -> 0 bytes .../array_1/compressor_1/.zarray | 17 - .../array_1/compressor_1/0 | Bin 916 -> 0 bytes .../array_1/compressor_1/1 | Bin 770 -> 0 bytes .../array_1/compressor_2/.zarray | 17 - .../array_1/compressor_2/0 | Bin 714 -> 0 bytes .../array_1/compressor_2/1 | Bin 667 -> 0 bytes .../array_1/compressor_3/.zarray | 20 - .../array_1/compressor_3/0 | Bin 945 -> 0 bytes .../array_1/compressor_3/1 | Bin 798 -> 0 bytes .../array_1/compressor_4/.zarray | 20 - .../array_1/compressor_4/0 | Bin 311 -> 0 bytes .../array_1/compressor_4/1 | Bin 318 -> 0 bytes .../array_1/compressor_5/.zarray | 20 - .../array_1/compressor_5/0 | Bin 138 -> 0 bytes .../array_1/compressor_5/1 | Bin 150 -> 0 bytes .../array_1/compressor_6/.zarray | 20 - .../array_1/compressor_6/0 | Bin 1216 -> 0 bytes .../array_1/compressor_6/1 | Bin 1071 -> 0 bytes .../array_10/.zgroup | 3 - .../array_10/compressor_0/.zarray | 14 - .../array_10/compressor_0/0 | Bin 16000 -> 0 bytes .../array_10/compressor_0/1 | Bin 16000 -> 0 bytes .../array_10/compressor_1/.zarray | 17 - .../array_10/compressor_1/0 | Bin 8287 -> 0 bytes .../array_10/compressor_1/1 | Bin 6176 -> 0 bytes .../array_10/compressor_2/.zarray | 17 - .../array_10/compressor_2/0 | Bin 7521 -> 0 bytes .../array_10/compressor_2/1 | Bin 4670 -> 0 bytes .../array_10/compressor_3/.zarray | 20 - .../array_10/compressor_3/0 | Bin 8172 -> 0 bytes .../array_10/compressor_3/1 | Bin 5436 -> 0 bytes .../array_10/compressor_4/.zarray | 20 - .../array_10/compressor_4/0 | Bin 13861 -> 0 bytes .../array_10/compressor_4/1 | Bin 4740 -> 0 bytes .../array_10/compressor_5/.zarray | 20 - .../array_10/compressor_5/0 | Bin 4127 -> 0 bytes .../array_10/compressor_5/1 | Bin 2492 -> 0 bytes .../array_10/compressor_6/.zarray | 20 - .../array_10/compressor_6/0 | Bin 15987 -> 0 bytes .../array_10/compressor_6/1 | Bin 10775 -> 0 bytes .../array_11/.zgroup | 3 - .../array_11/compressor_0/.zarray | 14 - .../array_11/compressor_0/0 | Bin 6000 -> 0 bytes .../array_11/compressor_0/1 | Bin 6000 -> 0 bytes .../array_11/compressor_1/.zarray | 17 - .../array_11/compressor_1/0 | Bin 5584 -> 0 bytes .../array_11/compressor_1/1 | Bin 2758 -> 0 bytes .../array_11/compressor_2/.zarray | 17 - .../array_11/compressor_2/0 | Bin 5888 -> 0 bytes .../array_11/compressor_2/1 | Bin 3083 -> 0 bytes .../array_11/compressor_3/.zarray | 20 - .../array_11/compressor_3/0 | Bin 5609 -> 0 bytes .../array_11/compressor_3/1 | Bin 2768 -> 0 bytes .../array_11/compressor_4/.zarray | 20 - .../array_11/compressor_4/0 | Bin 5609 -> 0 bytes .../array_11/compressor_4/1 | Bin 2778 -> 0 bytes .../array_11/compressor_5/.zarray | 20 - .../array_11/compressor_5/0 | Bin 5726 -> 0 bytes .../array_11/compressor_5/1 | Bin 2934 -> 0 bytes .../array_11/compressor_6/.zarray | 20 - .../array_11/compressor_6/0 | Bin 6016 -> 0 bytes .../array_11/compressor_6/1 | Bin 2973 -> 0 bytes .../array_12/.zgroup | 3 - .../array_12/compressor_0/.zarray | 14 - .../array_12/compressor_0/0 | Bin 12000 -> 0 bytes .../array_12/compressor_0/1 | Bin 12000 -> 0 bytes .../array_12/compressor_1/.zarray | 17 - .../array_12/compressor_1/0 | Bin 11144 -> 0 bytes .../array_12/compressor_1/1 | Bin 5453 -> 0 bytes .../array_12/compressor_2/.zarray | 17 - .../array_12/compressor_2/0 | Bin 12115 -> 0 bytes .../array_12/compressor_2/1 | Bin 6120 -> 0 bytes .../array_12/compressor_3/.zarray | 20 - .../array_12/compressor_3/0 | Bin 11146 -> 0 bytes .../array_12/compressor_3/1 | Bin 5435 -> 0 bytes .../array_12/compressor_4/.zarray | 20 - .../array_12/compressor_4/0 | Bin 11138 -> 0 bytes .../array_12/compressor_4/1 | Bin 5380 -> 0 bytes .../array_12/compressor_5/.zarray | 20 - .../array_12/compressor_5/0 | Bin 10803 -> 0 bytes .../array_12/compressor_5/1 | Bin 5295 -> 0 bytes .../array_12/compressor_6/.zarray | 20 - .../array_12/compressor_6/0 | Bin 12016 -> 0 bytes .../array_12/compressor_6/1 | Bin 5890 -> 0 bytes .../array_13/.zgroup | 3 - .../array_13/compressor_0/.zarray | 14 - .../array_13/compressor_0/0 | Bin 24000 -> 0 bytes .../array_13/compressor_0/1 | Bin 24000 -> 0 bytes .../array_13/compressor_1/.zarray | 17 - .../array_13/compressor_1/0 | Bin 23139 -> 0 bytes .../array_13/compressor_1/1 | Bin 11246 -> 0 bytes .../array_13/compressor_2/.zarray | 17 - .../array_13/compressor_2/0 | Bin 23896 -> 0 bytes .../array_13/compressor_2/1 | Bin 11779 -> 0 bytes .../array_13/compressor_3/.zarray | 20 - .../array_13/compressor_3/0 | Bin 23050 -> 0 bytes .../array_13/compressor_3/1 | Bin 11156 -> 0 bytes .../array_13/compressor_4/.zarray | 20 - .../array_13/compressor_4/0 | Bin 22820 -> 0 bytes .../array_13/compressor_4/1 | Bin 10725 -> 0 bytes .../array_13/compressor_5/.zarray | 20 - .../array_13/compressor_5/0 | Bin 21824 -> 0 bytes .../array_13/compressor_5/1 | Bin 10647 -> 0 bytes .../array_13/compressor_6/.zarray | 20 - .../array_13/compressor_6/0 | Bin 24016 -> 0 bytes .../array_13/compressor_6/1 | Bin 11725 -> 0 bytes .../array_14/.zgroup | 3 - .../array_14/compressor_0/.zarray | 14 - .../array_14/compressor_0/0 | 1 - .../array_14/compressor_0/1 | Bin 3000 -> 0 bytes .../array_14/compressor_1/.zarray | 17 - .../array_14/compressor_1/0 | Bin 1058 -> 0 bytes .../array_14/compressor_1/1 | Bin 918 -> 0 bytes .../array_14/compressor_2/.zarray | 17 - .../array_14/compressor_2/0 | Bin 888 -> 0 bytes .../array_14/compressor_2/1 | Bin 797 -> 0 bytes .../array_14/compressor_3/.zarray | 20 - .../array_14/compressor_3/0 | Bin 1016 -> 0 bytes .../array_14/compressor_3/1 | Bin 895 -> 0 bytes .../array_14/compressor_4/.zarray | 20 - .../array_14/compressor_4/0 | Bin 1016 -> 0 bytes .../array_14/compressor_4/1 | Bin 895 -> 0 bytes .../array_14/compressor_5/.zarray | 20 - .../array_14/compressor_5/0 | Bin 1559 -> 0 bytes .../array_14/compressor_5/1 | Bin 1349 -> 0 bytes .../array_14/compressor_6/.zarray | 20 - .../array_14/compressor_6/0 | Bin 2311 -> 0 bytes .../array_14/compressor_6/1 | Bin 2037 -> 0 bytes .../array_15/.zgroup | 3 - .../array_15/compressor_0/.zarray | 14 - .../array_15/compressor_0/0 | Bin 48000 -> 0 bytes .../array_15/compressor_0/1 | Bin 48000 -> 0 bytes .../array_15/compressor_1/.zarray | 17 - .../array_15/compressor_1/0 | 1 - .../array_15/compressor_1/1 | Bin 3139 -> 0 bytes .../array_15/compressor_2/.zarray | 17 - .../array_15/compressor_2/0 | Bin 1028 -> 0 bytes .../array_15/compressor_2/1 | Bin 906 -> 0 bytes .../array_15/compressor_3/.zarray | 20 - .../array_15/compressor_3/0 | Bin 3116 -> 0 bytes .../array_15/compressor_3/1 | Bin 2684 -> 0 bytes .../array_15/compressor_4/.zarray | 20 - .../array_15/compressor_4/0 | Bin 3623 -> 0 bytes .../array_15/compressor_4/1 | Bin 3130 -> 0 bytes .../array_15/compressor_5/.zarray | 20 - .../array_15/compressor_5/0 | Bin 2464 -> 0 bytes .../array_15/compressor_5/1 | Bin 2152 -> 0 bytes .../array_15/compressor_6/.zarray | 20 - .../array_15/compressor_6/0 | Bin 10106 -> 0 bytes .../array_15/compressor_6/1 | Bin 8644 -> 0 bytes .../array_16/.zgroup | 3 - .../array_16/compressor_0/.zarray | 14 - .../array_16/compressor_0/0 | Bin 24000 -> 0 bytes .../array_16/compressor_0/1 | Bin 24000 -> 0 bytes .../array_16/compressor_1/.zarray | 17 - .../array_16/compressor_1/0 | 1 - .../array_16/compressor_1/1 | Bin 2498 -> 0 bytes .../array_16/compressor_2/.zarray | 17 - .../array_16/compressor_2/0 | Bin 1252 -> 0 bytes .../array_16/compressor_2/1 | Bin 1108 -> 0 bytes .../array_16/compressor_3/.zarray | 20 - .../array_16/compressor_3/0 | Bin 3029 -> 0 bytes .../array_16/compressor_3/1 | Bin 2548 -> 0 bytes .../array_16/compressor_4/.zarray | 20 - .../array_16/compressor_4/0 | Bin 5423 -> 0 bytes .../array_16/compressor_4/1 | Bin 4698 -> 0 bytes .../array_16/compressor_5/.zarray | 20 - .../array_16/compressor_5/0 | Bin 3624 -> 0 bytes .../array_16/compressor_5/1 | Bin 3139 -> 0 bytes .../array_16/compressor_6/.zarray | 20 - .../array_16/compressor_6/0 | Bin 8229 -> 0 bytes .../array_16/compressor_6/1 | Bin 7158 -> 0 bytes .../array_17/.zgroup | 3 - .../array_17/compressor_0/.zarray | 14 - .../array_17/compressor_0/0 | Bin 3000 -> 0 bytes .../array_17/compressor_0/1 | Bin 3000 -> 0 bytes .../array_17/compressor_1/.zarray | 17 - .../array_17/compressor_1/0 | Bin 709 -> 0 bytes .../array_17/compressor_1/1 | Bin 615 -> 0 bytes .../array_17/compressor_2/.zarray | 17 - .../array_17/compressor_2/0 | Bin 537 -> 0 bytes .../array_17/compressor_2/1 | Bin 473 -> 0 bytes .../array_17/compressor_3/.zarray | 20 - .../array_17/compressor_3/0 | Bin 703 -> 0 bytes .../array_17/compressor_3/1 | Bin 608 -> 0 bytes .../array_17/compressor_4/.zarray | 20 - .../array_17/compressor_4/0 | Bin 703 -> 0 bytes .../array_17/compressor_4/1 | Bin 608 -> 0 bytes .../array_17/compressor_5/.zarray | 20 - .../array_17/compressor_5/0 | Bin 419 -> 0 bytes .../array_17/compressor_5/1 | Bin 366 -> 0 bytes .../array_17/compressor_6/.zarray | 20 - .../array_17/compressor_6/0 | Bin 1870 -> 0 bytes .../array_17/compressor_6/1 | Bin 1613 -> 0 bytes .../array_18/.zgroup | 3 - .../array_18/compressor_0/.zarray | 16 - .../array_18/compressor_0/0.0 | Bin 40000 -> 0 bytes .../array_18/compressor_0/0.1 | Bin 40000 -> 0 bytes .../array_18/compressor_1/.zarray | 19 - .../array_18/compressor_1/0.0 | Bin 13951 -> 0 bytes .../array_18/compressor_1/0.1 | Bin 13936 -> 0 bytes .../array_18/compressor_2/.zarray | 19 - .../array_18/compressor_2/0.0 | Bin 9114 -> 0 bytes .../array_18/compressor_2/0.1 | Bin 8672 -> 0 bytes .../array_18/compressor_3/.zarray | 22 - .../array_18/compressor_3/0.0 | Bin 23597 -> 0 bytes .../array_18/compressor_3/0.1 | Bin 23613 -> 0 bytes .../array_18/compressor_4/.zarray | 22 - .../array_18/compressor_4/0.0 | Bin 1607 -> 0 bytes .../array_18/compressor_4/0.1 | Bin 1607 -> 0 bytes .../array_18/compressor_5/.zarray | 22 - .../array_18/compressor_5/0.0 | Bin 682 -> 0 bytes .../array_18/compressor_5/0.1 | Bin 700 -> 0 bytes .../array_18/compressor_6/.zarray | 22 - .../array_18/compressor_6/0.0 | Bin 40016 -> 0 bytes .../array_18/compressor_6/0.1 | Bin 40016 -> 0 bytes .../array_19/.zgroup | 3 - .../array_19/compressor_0/.zarray | 16 - .../array_19/compressor_0/0.0 | Bin 40000 -> 0 bytes .../array_19/compressor_0/0.1 | Bin 40000 -> 0 bytes .../array_19/compressor_1/.zarray | 19 - .../array_19/compressor_1/0.0 | Bin 13895 -> 0 bytes .../array_19/compressor_1/0.1 | Bin 13864 -> 0 bytes .../array_19/compressor_2/.zarray | 19 - .../array_19/compressor_2/0.0 | Bin 3330 -> 0 bytes .../array_19/compressor_2/0.1 | Bin 3372 -> 0 bytes .../array_19/compressor_3/.zarray | 22 - .../array_19/compressor_3/0.0 | Bin 22581 -> 0 bytes .../array_19/compressor_3/0.1 | Bin 22801 -> 0 bytes .../array_19/compressor_4/.zarray | 22 - .../array_19/compressor_4/0.0 | Bin 743 -> 0 bytes .../array_19/compressor_4/0.1 | Bin 747 -> 0 bytes .../array_19/compressor_5/.zarray | 22 - .../array_19/compressor_5/0.0 | Bin 507 -> 0 bytes .../array_19/compressor_5/0.1 | Bin 508 -> 0 bytes .../array_19/compressor_6/.zarray | 22 - .../array_19/compressor_6/0.0 | Bin 40016 -> 0 bytes .../array_19/compressor_6/0.1 | Bin 40016 -> 0 bytes .../test_format_compatibility/array_2/.zgroup | 3 - .../array_2/compressor_0/.zarray | 14 - .../array_2/compressor_0/0 | Bin 2400 -> 0 bytes .../array_2/compressor_0/1 | Bin 2400 -> 0 bytes .../array_2/compressor_1/.zarray | 17 - .../array_2/compressor_1/0 | Bin 874 -> 0 bytes .../array_2/compressor_1/1 | Bin 762 -> 0 bytes .../array_2/compressor_2/.zarray | 17 - .../array_2/compressor_2/0 | Bin 760 -> 0 bytes .../array_2/compressor_2/1 | Bin 686 -> 0 bytes .../array_2/compressor_3/.zarray | 20 - .../array_2/compressor_3/0 | Bin 1133 -> 0 bytes .../array_2/compressor_3/1 | Bin 1055 -> 0 bytes .../array_2/compressor_4/.zarray | 20 - .../array_2/compressor_4/0 | Bin 315 -> 0 bytes .../array_2/compressor_4/1 | Bin 318 -> 0 bytes .../array_2/compressor_5/.zarray | 20 - .../array_2/compressor_5/0 | Bin 138 -> 0 bytes .../array_2/compressor_5/1 | Bin 151 -> 0 bytes .../array_2/compressor_6/.zarray | 20 - .../array_2/compressor_6/0 | Bin 2416 -> 0 bytes .../array_2/compressor_6/1 | Bin 2099 -> 0 bytes .../array_20/.zgroup | 3 - .../array_20/compressor_0/.zarray | 18 - .../array_20/compressor_0/0.0.0 | Bin 40000 -> 0 bytes .../array_20/compressor_0/0.0.1 | Bin 40000 -> 0 bytes .../array_20/compressor_1/.zarray | 21 - .../array_20/compressor_1/0.0.0 | Bin 13951 -> 0 bytes .../array_20/compressor_1/0.0.1 | Bin 13936 -> 0 bytes .../array_20/compressor_2/.zarray | 21 - .../array_20/compressor_2/0.0.0 | Bin 9114 -> 0 bytes .../array_20/compressor_2/0.0.1 | Bin 8672 -> 0 bytes .../array_20/compressor_3/.zarray | 24 - .../array_20/compressor_3/0.0.0 | Bin 23597 -> 0 bytes .../array_20/compressor_3/0.0.1 | Bin 23613 -> 0 bytes .../array_20/compressor_4/.zarray | 24 - .../array_20/compressor_4/0.0.0 | Bin 1607 -> 0 bytes .../array_20/compressor_4/0.0.1 | Bin 1607 -> 0 bytes .../array_20/compressor_5/.zarray | 24 - .../array_20/compressor_5/0.0.0 | Bin 682 -> 0 bytes .../array_20/compressor_5/0.0.1 | Bin 700 -> 0 bytes .../array_20/compressor_6/.zarray | 24 - .../array_20/compressor_6/0.0.0 | Bin 40016 -> 0 bytes .../array_20/compressor_6/0.0.1 | Bin 40016 -> 0 bytes .../array_21/.zgroup | 3 - .../array_21/compressor_0/.zarray | 18 - .../array_21/compressor_0/0.0.0 | Bin 80000 -> 0 bytes .../array_21/compressor_1/.zarray | 21 - .../array_21/compressor_1/0.0.0 | Bin 27715 -> 0 bytes .../array_21/compressor_2/.zarray | 21 - .../array_21/compressor_2/0.0.0 | Bin 6038 -> 0 bytes .../array_21/compressor_3/.zarray | 24 - .../array_21/compressor_3/0.0.0 | Bin 45710 -> 0 bytes .../array_21/compressor_4/.zarray | 24 - .../array_21/compressor_4/0.0.0 | Bin 1180 -> 0 bytes .../array_21/compressor_5/.zarray | 24 - .../array_21/compressor_5/0.0.0 | Bin 834 -> 0 bytes .../array_21/compressor_6/.zarray | 24 - .../array_21/compressor_6/0.0.0 | Bin 80016 -> 0 bytes .../array_22/.zgroup | 3 - .../array_22/compressor_0/.zarray | 20 - .../array_22/compressor_0/0.0.0.0 | Bin 40000 -> 0 bytes .../array_22/compressor_0/0.0.0.1 | Bin 40000 -> 0 bytes .../array_22/compressor_1/.zarray | 23 - .../array_22/compressor_1/0.0.0.0 | Bin 13951 -> 0 bytes .../array_22/compressor_1/0.0.0.1 | Bin 13936 -> 0 bytes .../array_22/compressor_2/.zarray | 23 - .../array_22/compressor_2/0.0.0.0 | Bin 9114 -> 0 bytes .../array_22/compressor_2/0.0.0.1 | Bin 8672 -> 0 bytes .../array_22/compressor_3/.zarray | 26 - .../array_22/compressor_3/0.0.0.0 | Bin 23597 -> 0 bytes .../array_22/compressor_3/0.0.0.1 | Bin 23613 -> 0 bytes .../array_22/compressor_4/.zarray | 26 - .../array_22/compressor_4/0.0.0.0 | Bin 1607 -> 0 bytes .../array_22/compressor_4/0.0.0.1 | Bin 1607 -> 0 bytes .../array_22/compressor_5/.zarray | 26 - .../array_22/compressor_5/0.0.0.0 | Bin 682 -> 0 bytes .../array_22/compressor_5/0.0.0.1 | Bin 700 -> 0 bytes .../array_22/compressor_6/.zarray | 26 - .../array_22/compressor_6/0.0.0.0 | Bin 40016 -> 0 bytes .../array_22/compressor_6/0.0.0.1 | Bin 40016 -> 0 bytes .../array_23/.zgroup | 3 - .../array_23/compressor_0/.zarray | 20 - .../array_23/compressor_0/0.0.0.0 | Bin 40000 -> 0 bytes .../array_23/compressor_0/0.0.0.1 | Bin 40000 -> 0 bytes .../array_23/compressor_1/.zarray | 23 - .../array_23/compressor_1/0.0.0.0 | Bin 13895 -> 0 bytes .../array_23/compressor_1/0.0.0.1 | Bin 13864 -> 0 bytes .../array_23/compressor_2/.zarray | 23 - .../array_23/compressor_2/0.0.0.0 | Bin 3330 -> 0 bytes .../array_23/compressor_2/0.0.0.1 | Bin 3372 -> 0 bytes .../array_23/compressor_3/.zarray | 26 - .../array_23/compressor_3/0.0.0.0 | Bin 22581 -> 0 bytes .../array_23/compressor_3/0.0.0.1 | Bin 22801 -> 0 bytes .../array_23/compressor_4/.zarray | 26 - .../array_23/compressor_4/0.0.0.0 | Bin 743 -> 0 bytes .../array_23/compressor_4/0.0.0.1 | Bin 747 -> 0 bytes .../array_23/compressor_5/.zarray | 26 - .../array_23/compressor_5/0.0.0.0 | Bin 507 -> 0 bytes .../array_23/compressor_5/0.0.0.1 | Bin 508 -> 0 bytes .../array_23/compressor_6/.zarray | 26 - .../array_23/compressor_6/0.0.0.0 | Bin 40016 -> 0 bytes .../array_23/compressor_6/0.0.0.1 | Bin 40016 -> 0 bytes .../test_format_compatibility/array_3/.zgroup | 3 - .../array_3/compressor_0/.zarray | 14 - .../array_3/compressor_0/0 | Bin 8000 -> 0 bytes .../array_3/compressor_0/1 | Bin 8000 -> 0 bytes .../array_3/compressor_1/.zarray | 17 - .../array_3/compressor_1/0 | Bin 1548 -> 0 bytes .../array_3/compressor_1/1 | Bin 252 -> 0 bytes .../array_3/compressor_2/.zarray | 17 - .../array_3/compressor_2/0 | Bin 1000 -> 0 bytes .../array_3/compressor_2/1 | Bin 220 -> 0 bytes .../array_3/compressor_3/.zarray | 20 - .../array_3/compressor_3/0 | Bin 1441 -> 0 bytes .../array_3/compressor_3/1 | Bin 178 -> 0 bytes .../array_3/compressor_4/.zarray | 20 - .../array_3/compressor_4/0 | Bin 320 -> 0 bytes .../array_3/compressor_4/1 | Bin 164 -> 0 bytes .../array_3/compressor_5/.zarray | 20 - .../array_3/compressor_5/0 | Bin 151 -> 0 bytes .../array_3/compressor_5/1 | Bin 117 -> 0 bytes .../array_3/compressor_6/.zarray | 20 - .../array_3/compressor_6/0 | Bin 4131 -> 0 bytes .../array_3/compressor_6/1 | Bin 612 -> 0 bytes .../test_format_compatibility/array_4/.zgroup | 3 - .../array_4/compressor_0/.zarray | 14 - .../array_4/compressor_0/0 | Bin 1200 -> 0 bytes .../array_4/compressor_0/1 | Bin 1200 -> 0 bytes .../array_4/compressor_1/.zarray | 17 - .../array_4/compressor_1/0 | Bin 1196 -> 0 bytes .../array_4/compressor_1/1 | Bin 1033 -> 0 bytes .../array_4/compressor_2/.zarray | 17 - .../array_4/compressor_2/0 | Bin 1417 -> 0 bytes .../array_4/compressor_2/1 | Bin 1195 -> 0 bytes .../array_4/compressor_3/.zarray | 20 - .../array_4/compressor_3/0 | Bin 1216 -> 0 bytes .../array_4/compressor_3/1 | Bin 1066 -> 0 bytes .../array_4/compressor_4/.zarray | 20 - .../array_4/compressor_4/0 | Bin 1216 -> 0 bytes .../array_4/compressor_4/1 | Bin 1066 -> 0 bytes .../array_4/compressor_5/.zarray | 20 - .../array_4/compressor_5/0 | Bin 1216 -> 0 bytes .../array_4/compressor_5/1 | Bin 1081 -> 0 bytes .../array_4/compressor_6/.zarray | 20 - .../array_4/compressor_6/0 | Bin 1216 -> 0 bytes .../array_4/compressor_6/1 | Bin 1070 -> 0 bytes .../test_format_compatibility/array_5/.zgroup | 3 - .../array_5/compressor_0/.zarray | 14 - .../array_5/compressor_0/0 | Bin 2400 -> 0 bytes .../array_5/compressor_0/1 | Bin 2400 -> 0 bytes .../array_5/compressor_1/.zarray | 17 - .../array_5/compressor_1/0 | Bin 1998 -> 0 bytes .../array_5/compressor_1/1 | Bin 1724 -> 0 bytes .../array_5/compressor_2/.zarray | 17 - .../array_5/compressor_2/0 | Bin 2030 -> 0 bytes .../array_5/compressor_2/1 | Bin 1809 -> 0 bytes .../array_5/compressor_3/.zarray | 20 - .../array_5/compressor_3/0 | Bin 2010 -> 0 bytes .../array_5/compressor_3/1 | Bin 1740 -> 0 bytes .../array_5/compressor_4/.zarray | 20 - .../array_5/compressor_4/0 | Bin 2010 -> 0 bytes .../array_5/compressor_4/1 | Bin 1743 -> 0 bytes .../array_5/compressor_5/.zarray | 20 - .../array_5/compressor_5/0 | Bin 1700 -> 0 bytes .../array_5/compressor_5/1 | Bin 1478 -> 0 bytes .../array_5/compressor_6/.zarray | 20 - .../array_5/compressor_6/0 | Bin 2416 -> 0 bytes .../array_5/compressor_6/1 | Bin 2097 -> 0 bytes .../test_format_compatibility/array_6/.zgroup | 3 - .../array_6/compressor_0/.zarray | 14 - .../array_6/compressor_0/0 | Bin 4800 -> 0 bytes .../array_6/compressor_0/1 | Bin 4800 -> 0 bytes .../array_6/compressor_1/.zarray | 17 - .../array_6/compressor_1/0 | Bin 2588 -> 0 bytes .../array_6/compressor_1/1 | Bin 2231 -> 0 bytes .../array_6/compressor_2/.zarray | 17 - .../array_6/compressor_2/0 | Bin 2094 -> 0 bytes .../array_6/compressor_2/1 | Bin 1824 -> 0 bytes .../array_6/compressor_3/.zarray | 20 - .../array_6/compressor_3/0 | Bin 2471 -> 0 bytes .../array_6/compressor_3/1 | Bin 2139 -> 0 bytes .../array_6/compressor_4/.zarray | 20 - .../array_6/compressor_4/0 | Bin 2014 -> 0 bytes .../array_6/compressor_4/1 | Bin 1742 -> 0 bytes .../array_6/compressor_5/.zarray | 20 - .../array_6/compressor_5/0 | Bin 1700 -> 0 bytes .../array_6/compressor_5/1 | Bin 1479 -> 0 bytes .../array_6/compressor_6/.zarray | 20 - .../array_6/compressor_6/0 | Bin 4816 -> 0 bytes .../array_6/compressor_6/1 | Bin 4145 -> 0 bytes .../test_format_compatibility/array_7/.zgroup | 3 - .../array_7/compressor_0/.zarray | 14 - .../array_7/compressor_0/0 | Bin 9600 -> 0 bytes .../array_7/compressor_0/1 | Bin 9600 -> 0 bytes .../array_7/compressor_1/.zarray | 17 - .../array_7/compressor_1/0 | 7 - .../array_7/compressor_1/1 | Bin 2422 -> 0 bytes .../array_7/compressor_2/.zarray | 17 - .../array_7/compressor_2/0 | Bin 2267 -> 0 bytes .../array_7/compressor_2/1 | Bin 1955 -> 0 bytes .../array_7/compressor_3/.zarray | 20 - .../array_7/compressor_3/0 | Bin 2144 -> 0 bytes .../array_7/compressor_3/1 | Bin 1833 -> 0 bytes .../array_7/compressor_4/.zarray | 20 - .../array_7/compressor_4/0 | Bin 2021 -> 0 bytes .../array_7/compressor_4/1 | Bin 1745 -> 0 bytes .../array_7/compressor_5/.zarray | 20 - .../array_7/compressor_5/0 | Bin 1700 -> 0 bytes .../array_7/compressor_5/1 | Bin 1479 -> 0 bytes .../array_7/compressor_6/.zarray | 20 - .../array_7/compressor_6/0 | Bin 5006 -> 0 bytes .../array_7/compressor_6/1 | Bin 4276 -> 0 bytes .../test_format_compatibility/array_8/.zgroup | 3 - .../array_8/compressor_0/.zarray | 14 - .../array_8/compressor_0/0 | Bin 4000 -> 0 bytes .../array_8/compressor_0/1 | Bin 4000 -> 0 bytes .../array_8/compressor_1/.zarray | 17 - .../array_8/compressor_1/0 | Bin 3291 -> 0 bytes .../array_8/compressor_1/1 | Bin 1336 -> 0 bytes .../array_8/compressor_2/.zarray | 17 - .../array_8/compressor_2/0 | Bin 2993 -> 0 bytes .../array_8/compressor_2/1 | Bin 1128 -> 0 bytes .../array_8/compressor_3/.zarray | 20 - .../array_8/compressor_3/0 | Bin 3477 -> 0 bytes .../array_8/compressor_3/1 | Bin 2055 -> 0 bytes .../array_8/compressor_4/.zarray | 20 - .../array_8/compressor_4/0 | Bin 1915 -> 0 bytes .../array_8/compressor_4/1 | Bin 900 -> 0 bytes .../array_8/compressor_5/.zarray | 20 - .../array_8/compressor_5/0 | Bin 735 -> 0 bytes .../array_8/compressor_5/1 | Bin 404 -> 0 bytes .../array_8/compressor_6/.zarray | 20 - .../array_8/compressor_6/0 | Bin 4016 -> 0 bytes .../array_8/compressor_6/1 | Bin 2722 -> 0 bytes .../test_format_compatibility/array_9/.zgroup | 3 - .../array_9/compressor_0/.zarray | 14 - .../array_9/compressor_0/0 | Bin 8000 -> 0 bytes .../array_9/compressor_0/1 | Bin 8000 -> 0 bytes .../array_9/compressor_1/.zarray | 17 - .../array_9/compressor_1/0 | Bin 6736 -> 0 bytes .../array_9/compressor_1/1 | Bin 4349 -> 0 bytes .../array_9/compressor_2/.zarray | 17 - .../array_9/compressor_2/0 | Bin 5805 -> 0 bytes .../array_9/compressor_2/1 | Bin 3480 -> 0 bytes .../array_9/compressor_3/.zarray | 20 - .../array_9/compressor_3/0 | Bin 7248 -> 0 bytes .../array_9/compressor_3/1 | Bin 4581 -> 0 bytes .../array_9/compressor_4/.zarray | 20 - .../array_9/compressor_4/0 | Bin 5900 -> 0 bytes .../array_9/compressor_4/1 | Bin 1810 -> 0 bytes .../array_9/compressor_5/.zarray | 20 - .../array_9/compressor_5/0 | Bin 1813 -> 0 bytes .../array_9/compressor_5/1 | Bin 1062 -> 0 bytes .../array_9/compressor_6/.zarray | 20 - .../array_9/compressor_6/0 | Bin 8016 -> 0 bytes .../array_9/compressor_6/1 | Bin 5415 -> 0 bytes tests/v2/test_attrs.py | 265 -- tests/v2/test_convenience.py | 804 ----- tests/v2/test_core.py | 2510 --------------- tests/v2/test_creation.py | 691 ---- tests/v2/test_dim_separator.py | 136 - tests/v2/test_filters.py | 209 -- tests/v2/test_hierarchy.py | 1452 --------- tests/v2/test_indexing.py | 1758 ---------- tests/v2/test_info.py | 66 - tests/v2/test_meta.py | 527 --- tests/v2/test_meta_array.py | 251 -- tests/v2/test_n5.py | 53 - tests/v2/test_storage.py | 2505 --------------- tests/v2/test_storage_v3.py | 692 ---- tests/v2/test_sync.py | 319 -- tests/v2/test_util.py | 292 -- tests/v2/util.py | 91 - tests/v3/test_codecs/test_codecs.py | 101 - tests/v3/test_codecs/test_endian.py | 29 - tests/v3/test_codecs/test_transpose.py | 17 - 572 files changed, 29200 deletions(-) delete mode 100644 src/zarr/v2/__init__.py delete mode 100644 src/zarr/v2/_storage/__init__.py delete mode 100644 src/zarr/v2/_storage/absstore.py delete mode 100644 src/zarr/v2/_storage/store.py delete mode 100644 src/zarr/v2/attrs.py delete mode 100644 src/zarr/v2/codecs.py delete mode 100644 src/zarr/v2/context.py delete mode 100644 src/zarr/v2/convenience.py delete mode 100644 src/zarr/v2/core.py delete mode 100644 src/zarr/v2/creation.py delete mode 100644 src/zarr/v2/errors.py delete mode 100644 src/zarr/v2/hierarchy.py delete mode 100644 src/zarr/v2/indexing.py delete mode 100644 src/zarr/v2/meta.py delete mode 100644 src/zarr/v2/meta_v1.py delete mode 100644 src/zarr/v2/n5.py delete mode 100644 src/zarr/v2/storage.py delete mode 100644 src/zarr/v2/sync.py delete mode 100644 src/zarr/v2/util.py delete mode 100644 tests/v2/__init__.py delete mode 100644 tests/v2/conftest.py delete mode 100644 tests/v2/fixture/.zgroup delete mode 100644 tests/v2/fixture/dimension_separator/flat/.zarray delete mode 100644 tests/v2/fixture/dimension_separator/flat/0.0 delete mode 100644 tests/v2/fixture/dimension_separator/flat_legacy/.zarray delete mode 100644 tests/v2/fixture/dimension_separator/flat_legacy/0.0 delete mode 100644 tests/v2/fixture/dimension_separator/nested/.zarray delete mode 100644 tests/v2/fixture/dimension_separator/nested/0/0 delete mode 100644 tests/v2/fixture/dimension_separator/nested_legacy/.zarray delete mode 100644 tests/v2/fixture/dimension_separator/nested_legacy/0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_0/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_1/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_10/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_11/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_12/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_13/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_14/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_15/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_16/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_17/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_0/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_0/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_1/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_1/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_2/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_2/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_3/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_3/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_4/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_4/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_5/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_5/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_6/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_18/compressor_6/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_0/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_0/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_1/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_1/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_2/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_2/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_3/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_3/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_4/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_4/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_5/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_5/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_6/0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_19/compressor_6/0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_2/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_0/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_0/0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_1/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_1/0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_2/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_2/0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_3/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_3/0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_4/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_4/0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_5/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_5/0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_6/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_20/compressor_6/0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_0/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_1/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_2/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_3/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_4/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_5/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_21/compressor_6/0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_0/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_0/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_1/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_1/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_2/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_2/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_3/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_3/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_4/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_4/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_5/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_5/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_6/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_22/compressor_6/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_0/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_0/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_1/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_1/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_2/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_2/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_3/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_3/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_4/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_4/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_5/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_5/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_6/0.0.0.0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_23/compressor_6/0.0.0.1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_3/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_4/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_5/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_6/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_7/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_8/compressor_6/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/.zgroup delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_0/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_0/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_0/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_1/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_1/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_1/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_2/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_2/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_2/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_3/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_3/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_3/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_4/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_4/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_4/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_5/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_5/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_5/1 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_6/.zarray delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_6/0 delete mode 100644 tests/v2/fixture/test_format_compatibility/array_9/compressor_6/1 delete mode 100644 tests/v2/test_attrs.py delete mode 100644 tests/v2/test_convenience.py delete mode 100644 tests/v2/test_core.py delete mode 100644 tests/v2/test_creation.py delete mode 100644 tests/v2/test_dim_separator.py delete mode 100644 tests/v2/test_filters.py delete mode 100644 tests/v2/test_hierarchy.py delete mode 100644 tests/v2/test_indexing.py delete mode 100644 tests/v2/test_info.py delete mode 100644 tests/v2/test_meta.py delete mode 100644 tests/v2/test_meta_array.py delete mode 100644 tests/v2/test_n5.py delete mode 100644 tests/v2/test_storage.py delete mode 100644 tests/v2/test_storage_v3.py delete mode 100644 tests/v2/test_sync.py delete mode 100644 tests/v2/test_util.py delete mode 100644 tests/v2/util.py diff --git a/src/zarr/v2/__init__.py b/src/zarr/v2/__init__.py deleted file mode 100644 index 27c7595580..0000000000 --- a/src/zarr/v2/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -# flake8: noqa -from zarr.v2.codecs import * -from zarr.v2.convenience import ( - consolidate_metadata, - copy, - copy_all, - copy_store, - load, - open, - open_consolidated, - save, - save_array, - save_group, - tree, -) -from zarr.v2.core import Array -from zarr.v2.creation import ( - array, - create, - empty, - empty_like, - full, - full_like, - ones, - ones_like, - open_array, - open_like, - zeros, - zeros_like, -) -from zarr.v2.errors import CopyError, MetadataError -from zarr.v2.hierarchy import Group, group, open_group -from zarr.v2.n5 import N5Store, N5FSStore -from zarr.v2.storage import ( - ABSStore, - DBMStore, - DictStore, - DirectoryStore, - KVStore, - LMDBStore, - LRUStoreCache, - MemoryStore, - MongoDBStore, - NestedDirectoryStore, - RedisStore, - SQLiteStore, - TempStore, - ZipStore, -) -from zarr.v2.sync import ProcessSynchronizer, ThreadSynchronizer -from zarr._version import version as __version__ - -# in case setuptools scm screw up and find version to be 0.0.0 -assert not __version__.startswith("0.0.0") diff --git a/src/zarr/v2/_storage/__init__.py b/src/zarr/v2/_storage/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/zarr/v2/_storage/absstore.py b/src/zarr/v2/_storage/absstore.py deleted file mode 100644 index c04ad240da..0000000000 --- a/src/zarr/v2/_storage/absstore.py +++ /dev/null @@ -1,224 +0,0 @@ -"""This module contains storage classes related to Azure Blob Storage (ABS)""" - -import warnings -from numcodecs.compat import ensure_bytes -from zarr.v2.util import normalize_storage_path -from zarr.v2._storage.store import Store - -__doctest_requires__ = { - ("ABSStore", "ABSStore.*"): ["azure.storage.blob"], -} - - -class ABSStore(Store): - """Storage class using Azure Blob Storage (ABS). - - Parameters - ---------- - container : string - The name of the ABS container to use. - - .. deprecated:: - Use ``client`` instead. - - prefix : string - Location of the "directory" to use as the root of the storage hierarchy - within the container. - - account_name : string - The Azure blob storage account name. - - .. deprecated:: 2.8.3 - Use ``client`` instead. - - account_key : string - The Azure blob storage account access key. - - .. deprecated:: 2.8.3 - Use ``client`` instead. - - blob_service_kwargs : dictionary - Extra arguments to be passed into the azure blob client, for e.g. when - using the emulator, pass in blob_service_kwargs={'is_emulated': True}. - - .. deprecated:: 2.8.3 - Use ``client`` instead. - - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - client : azure.storage.blob.ContainerClient, optional - And ``azure.storage.blob.ContainerClient`` to connect with. See - `here `_ # noqa - for more. - - .. versionadded:: 2.8.3 - - Notes - ----- - In order to use this store, you must install the Microsoft Azure Storage SDK for Python, - ``azure-storage-blob>=12.5.0``. - """ - - def __init__( - self, - container=None, - prefix="", - account_name=None, - account_key=None, - blob_service_kwargs=None, - dimension_separator=None, - client=None, - ): - self._dimension_separator = dimension_separator - self.prefix = normalize_storage_path(prefix) - if client is None: - # deprecated option, try to construct the client for them - msg = ( - "Providing 'container', 'account_name', 'account_key', and 'blob_service_kwargs'" - "is deprecated. Provide and instance of 'azure.storage.blob.ContainerClient' " - "'client' instead." - ) - warnings.warn(msg, FutureWarning, stacklevel=2) - from azure.storage.blob import ContainerClient - - blob_service_kwargs = blob_service_kwargs or {} - client = ContainerClient( - "https://{}.blob.core.windows.net/".format(account_name), - container, - credential=account_key, - **blob_service_kwargs, - ) - - self.client = client - self._container = container - self._account_name = account_name - self._account_key = account_key - - @staticmethod - def _warn_deprecated(property_): - msg = ( - "The {} property is deprecated and will be removed in a future " - "version. Get the property from 'ABSStore.client' instead." - ) - warnings.warn(msg.format(property_), FutureWarning, stacklevel=3) - - @property - def container(self): - self._warn_deprecated("container") - return self._container - - @property - def account_name(self): - self._warn_deprecated("account_name") - return self._account_name - - @property - def account_key(self): - self._warn_deprecated("account_key") - return self._account_key - - def _append_path_to_prefix(self, path): - if self.prefix == "": - return normalize_storage_path(path) - else: - return "/".join([self.prefix, normalize_storage_path(path)]) - - @staticmethod - def _strip_prefix_from_path(path, prefix): - # normalized things will not have any leading or trailing slashes - path_norm = normalize_storage_path(path) - prefix_norm = normalize_storage_path(prefix) - if prefix: - return path_norm[(len(prefix_norm) + 1) :] - else: - return path_norm - - def __getitem__(self, key): - from azure.core.exceptions import ResourceNotFoundError - - blob_name = self._append_path_to_prefix(key) - try: - return self.client.download_blob(blob_name).readall() - except ResourceNotFoundError: - raise KeyError("Blob %s not found" % blob_name) - - def __setitem__(self, key, value): - value = ensure_bytes(value) - blob_name = self._append_path_to_prefix(key) - self.client.upload_blob(blob_name, value, overwrite=True) - - def __delitem__(self, key): - from azure.core.exceptions import ResourceNotFoundError - - try: - self.client.delete_blob(self._append_path_to_prefix(key)) - except ResourceNotFoundError: - raise KeyError("Blob %s not found" % key) - - def __eq__(self, other): - return ( - isinstance(other, ABSStore) - and self.client == other.client - and self.prefix == other.prefix - ) - - def keys(self): - return list(self.__iter__()) - - def __iter__(self): - if self.prefix: - list_blobs_prefix = self.prefix + "/" - else: - list_blobs_prefix = None - for blob in self.client.list_blobs(list_blobs_prefix): - yield self._strip_prefix_from_path(blob.name, self.prefix) - - def __len__(self): - return len(self.keys()) - - def __contains__(self, key): - blob_name = self._append_path_to_prefix(key) - return self.client.get_blob_client(blob_name).exists() - - def listdir(self, path=None): - dir_path = normalize_storage_path(self._append_path_to_prefix(path)) - if dir_path: - dir_path += "/" - items = [ - self._strip_prefix_from_path(blob.name, dir_path) - for blob in self.client.walk_blobs(name_starts_with=dir_path, delimiter="/") - ] - return items - - def rmdir(self, path=None): - dir_path = normalize_storage_path(self._append_path_to_prefix(path)) - if dir_path: - dir_path += "/" - for blob in self.client.list_blobs(name_starts_with=dir_path): - self.client.delete_blob(blob) - - def getsize(self, path=None): - store_path = normalize_storage_path(path) - fs_path = self._append_path_to_prefix(store_path) - if fs_path: - blob_client = self.client.get_blob_client(fs_path) - else: - blob_client = None - - if blob_client and blob_client.exists(): - return blob_client.get_blob_properties().size - else: - size = 0 - if fs_path == "": - fs_path = None - elif not fs_path.endswith("/"): - fs_path += "/" - for blob in self.client.walk_blobs(name_starts_with=fs_path, delimiter="/"): - blob_client = self.client.get_blob_client(blob) - if blob_client.exists(): - size += blob_client.get_blob_properties().size - return size - - def clear(self): - self.rmdir() diff --git a/src/zarr/v2/_storage/store.py b/src/zarr/v2/_storage/store.py deleted file mode 100644 index ec1dbf0565..0000000000 --- a/src/zarr/v2/_storage/store.py +++ /dev/null @@ -1,226 +0,0 @@ -from collections.abc import MutableMapping -from typing import Any, List, Mapping, Optional, Sequence, Union - -from zarr.v2.meta import Metadata2 -from zarr.v2.util import normalize_storage_path -from zarr.v2.context import Context - - -# v2 store keys -array_meta_key = ".zarray" -group_meta_key = ".zgroup" -attrs_key = ".zattrs" - -DEFAULT_ZARR_VERSION = 2 - - -class BaseStore(MutableMapping[str, Any]): - """Abstract base class for store implementations. - - This is a thin wrapper over MutableMapping that provides methods to check - whether a store is readable, writeable, eraseable and or listable. - - Stores cannot be mutable mapping as they do have a couple of other - requirements that would break Liskov substitution principle (stores only - allow strings as keys, mutable mapping are more generic). - - Having no-op base method also helps simplifying store usage and do not need - to check the presence of attributes and methods, like `close()`. - - Stores can be used as context manager to make sure they close on exit. - - .. added: 2.11.0 - - """ - - _readable = True - _writeable = True - _erasable = True - _listable = True - _store_version = 2 - _metadata_class = Metadata2 - - def is_readable(self): - return self._readable - - def is_writeable(self): - return self._writeable - - def is_listable(self): - return self._listable - - def is_erasable(self): - return self._erasable - - def __enter__(self): - if not hasattr(self, "_open_count"): - self._open_count = 0 - self._open_count += 1 - return self - - def __exit__(self, exc_type, exc_value, traceback): - self._open_count -= 1 - if self._open_count == 0: - self.close() - - def close(self) -> None: - """Do nothing by default""" - pass - - def rename(self, src_path: str, dst_path: str) -> None: - if not self.is_erasable(): - raise NotImplementedError( - f'{type(self)} is not erasable, cannot call "rename"' - ) # pragma: no cover - _rename_from_keys(self, src_path, dst_path) - - @staticmethod - def _ensure_store(store: Any): - """ - We want to make sure internally that zarr stores are always a class - with a specific interface derived from ``BaseStore``, which is slightly - different than ``MutableMapping``. - - We'll do this conversion in a few places automatically - """ - from zarr.v2.storage import KVStore # avoid circular import - - if isinstance(store, BaseStore): - if not store._store_version == 2: - raise ValueError( - f"cannot initialize a v2 store with a v{store._store_version} store" - ) - return store - elif isinstance(store, MutableMapping): - return KVStore(store) - else: - for attr in [ - "keys", - "values", - "get", - "__setitem__", - "__getitem__", - "__delitem__", - "__contains__", - ]: - if not hasattr(store, attr): - break - else: - return KVStore(store) - - raise ValueError( - "Starting with Zarr 2.11.0, stores must be subclasses of " - "BaseStore, if your store exposes the MutableMapping interface " - f"wrap it in zarr.v2.storage.KVStore. Got {store}" - ) - - def getitems( - self, keys: Sequence[str], *, contexts: Mapping[str, Context] - ) -> Mapping[str, Any]: - """Retrieve data from multiple keys. - - Parameters - ---------- - keys : Iterable[str] - The keys to retrieve - contexts: Mapping[str, Context] - A mapping of keys to their context. Each context is a mapping of store - specific information. E.g. a context could be a dict telling the store - the preferred output array type: `{"meta_array": cupy.empty(())}` - - Returns - ------- - Mapping - A collection mapping the input keys to their results. - - Notes - ----- - This default implementation uses __getitem__() to read each key sequentially and - ignores contexts. Overwrite this method to implement concurrent reads of multiple - keys and/or to utilize the contexts. - """ - return {k: self[k] for k in keys if k in self} - - -class Store(BaseStore): - """Abstract store class used by implementations following the Zarr v2 spec. - - Adds public `listdir`, `rename`, and `rmdir` methods on top of BaseStore. - - .. added: 2.11.0 - - """ - - def listdir(self, path: str = "") -> List[str]: - path = normalize_storage_path(path) - return _listdir_from_keys(self, path) - - def rmdir(self, path: str = "") -> None: - if not self.is_erasable(): - raise NotImplementedError( - f'{type(self)} is not erasable, cannot call "rmdir"' - ) # pragma: no cover - path = normalize_storage_path(path) - _rmdir_from_keys(self, path) - - -# allow MutableMapping for backwards compatibility -StoreLike = Union[BaseStore, MutableMapping[str, Any]] - - -def _path_to_prefix(path: Optional[str]) -> str: - # assume path already normalized - if path: - prefix = path + "/" - else: - prefix = "" - return prefix - - -def _rename_from_keys(store: BaseStore, src_path: str, dst_path: str) -> None: - # assume path already normalized - src_prefix = _path_to_prefix(src_path) - dst_prefix = _path_to_prefix(dst_path) - version = getattr(store, "_store_version", 2) - if version == 2: - for key in list(store.keys()): - if key.startswith(src_prefix): - new_key = dst_prefix + key.lstrip(src_prefix) - store[new_key] = store.pop(key) - else: - raise NotImplementedError("This function only supports Zarr version 2.") - - -def _rmdir_from_keys(store: StoreLike, path: Optional[str] = None) -> None: - # assume path already normalized - prefix = _path_to_prefix(path) - for key in list(store.keys()): - if key.startswith(prefix): - del store[key] - - -def _listdir_from_keys(store: BaseStore, path: Optional[str] = None) -> List[str]: - # assume path already normalized - prefix = _path_to_prefix(path) - children = set() - for key in list(store.keys()): - if key.startswith(prefix) and len(key) > len(prefix): - suffix = key[len(prefix) :] - child = suffix.split("/")[0] - children.add(child) - return sorted(children) - - -def _prefix_to_array_key(store: StoreLike, prefix: str) -> str: - key = prefix + array_meta_key - return key - - -def _prefix_to_group_key(store: StoreLike, prefix: str) -> str: - key = prefix + group_meta_key - return key - - -def _prefix_to_attrs_key(store: StoreLike, prefix: str) -> str: - key = prefix + attrs_key - return key diff --git a/src/zarr/v2/attrs.py b/src/zarr/v2/attrs.py deleted file mode 100644 index af23d43b9e..0000000000 --- a/src/zarr/v2/attrs.py +++ /dev/null @@ -1,158 +0,0 @@ -from typing import Any -import warnings -from collections.abc import MutableMapping - -from zarr.v2._storage.store import Store -from zarr.v2.util import json_dumps - - -class Attributes(MutableMapping[str, Any]): - """Class providing access to user attributes on an array or group. Should not be - instantiated directly, will be available via the `.attrs` property of an array or - group. - - Parameters - ---------- - store : MutableMapping - The store in which to store the attributes. - key : str, optional - The key under which the attributes will be stored. - read_only : bool, optional - If True, attributes cannot be modified. - cache : bool, optional - If True (default), attributes will be cached locally. - synchronizer : Synchronizer - Only necessary if attributes may be modified from multiple threads or processes. - - """ - - def __init__(self, store, key=".zattrs", read_only=False, cache=True, synchronizer=None): - _Store = Store - self.store = _Store._ensure_store(store) - self.key = key - self.read_only = read_only - self.cache = cache - self._cached_asdict = None - self.synchronizer = synchronizer - - def _get_nosync(self): - try: - data = self.store[self.key] - except KeyError: - d: dict[str, Any] = dict() - else: - d = self.store._metadata_class.parse_metadata(data) - return d - - def asdict(self): - """Retrieve all attributes as a dictionary.""" - if self.cache and self._cached_asdict is not None: - return self._cached_asdict - d = self._get_nosync() - if self.cache: - self._cached_asdict = d - return d - - def refresh(self): - """Refresh cached attributes from the store.""" - if self.cache: - self._cached_asdict = self._get_nosync() - - def __contains__(self, x): - return x in self.asdict() - - def __getitem__(self, item): - return self.asdict()[item] - - def _write_op(self, f, *args, **kwargs): - # guard condition - if self.read_only: - raise PermissionError("attributes are read-only") - - # synchronization - if self.synchronizer is None: - return f(*args, **kwargs) - else: - with self.synchronizer[self.key]: - return f(*args, **kwargs) - - def __setitem__(self, item, value): - self._write_op(self._setitem_nosync, item, value) - - def _setitem_nosync(self, item, value): - # load existing data - d = self._get_nosync() - - # set key value - - d[item] = value - - # _put modified data - self._put_nosync(d) - - def __delitem__(self, item): - self._write_op(self._delitem_nosync, item) - - def _delitem_nosync(self, key): - # load existing data - d = self._get_nosync() - - # delete key value - del d[key] - - # _put modified data - self._put_nosync(d) - - def put(self, d): - """Overwrite all attributes with the key/value pairs in the provided dictionary - `d` in a single operation.""" - self._write_op(self._put_nosync, d) - - def _put_nosync(self, d): - d_to_check = d - if not all(isinstance(item, str) for item in d_to_check): - # TODO: Raise an error for non-string keys - # raise TypeError("attribute keys must be strings") - warnings.warn( - "only attribute keys of type 'string' will be allowed in the future", - DeprecationWarning, - stacklevel=2, - ) - - try: - d_to_check = {str(k): v for k, v in d_to_check.items()} - except TypeError as ex: # pragma: no cover - raise TypeError("attribute keys can not be stringified") from ex - - d = d_to_check - - self.store[self.key] = json_dumps(d) - if self.cache: - self._cached_asdict = d - - # noinspection PyMethodOverriding - def update(self, *args, **kwargs): - """Update the values of several attributes in a single operation.""" - self._write_op(self._update_nosync, *args, **kwargs) - - def _update_nosync(self, *args, **kwargs): - # load existing data - d = self._get_nosync() - - # update - d.update(*args, **kwargs) - - # _put modified data - self._put_nosync(d) - - def keys(self): - return self.asdict().keys() - - def __iter__(self): - return iter(self.asdict()) - - def __len__(self): - return len(self.asdict()) - - def _ipython_key_completions_(self): - return sorted(self) diff --git a/src/zarr/v2/codecs.py b/src/zarr/v2/codecs.py deleted file mode 100644 index 4ad68b8627..0000000000 --- a/src/zarr/v2/codecs.py +++ /dev/null @@ -1,4 +0,0 @@ -# flake8: noqa -from numcodecs import * -from numcodecs import get_codec, Blosc, Pickle, Zlib, Delta, AsType, BZ2 -from numcodecs.registry import codec_registry diff --git a/src/zarr/v2/context.py b/src/zarr/v2/context.py deleted file mode 100644 index 4eb1db7491..0000000000 --- a/src/zarr/v2/context.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import TypedDict - -from numcodecs.compat import NDArrayLike - - -class Context(TypedDict, total=False): - """A context for component specific information - - All keys are optional. Any component reading the context must provide - a default implementation in the case a key cannot be found. - - Attributes - ---------- - meta_array : array-like, optional - An array-like instance to use for determining the preferred output - array type. - """ - - meta_array: NDArrayLike diff --git a/src/zarr/v2/convenience.py b/src/zarr/v2/convenience.py deleted file mode 100644 index c066ee59e0..0000000000 --- a/src/zarr/v2/convenience.py +++ /dev/null @@ -1,1284 +0,0 @@ -"""Convenience functions for storing and loading data.""" - -import itertools -import os -import re -from collections.abc import Mapping, MutableMapping -from zarr.v2.core import Array -from zarr.v2.creation import array as _create_array -from zarr.v2.creation import open_array -from zarr.v2.errors import CopyError, PathNotFoundError -from zarr.v2.hierarchy import Group -from zarr.v2.hierarchy import group as _create_group -from zarr.v2.hierarchy import open_group -from zarr.v2.meta import json_dumps, json_loads -from zarr.v2.storage import ( - contains_array, - contains_group, - normalize_store_arg, - BaseStore, - ConsolidatedMetadataStore, -) -from zarr.v2.util import TreeViewer, buffer_size, normalize_storage_path - -from typing import Any, Union - -StoreLike = Union[BaseStore, MutableMapping[str, Any], str, None] - -_builtin_open = open # builtin open is later shadowed by a local open function - - -def _check_and_update_path(store: BaseStore, path): - if getattr(store, "_store_version", 2) > 2 and not path: - raise ValueError("path must be provided for v3 stores") - return normalize_storage_path(path) - - -# noinspection PyShadowingBuiltins -def open(store: StoreLike = None, mode: str = "a", *, path=None, **kwargs): - """Convenience function to open a group or array using file-mode-like semantics. - - Parameters - ---------- - store : Store or string, optional - Store or path to directory in file system or name of zip file. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist); 'a' means read/write (create if doesn't - exist); 'w' means create (overwrite if exists); 'w-' means create - (fail if exists). - path : str or None, optional - The path within the store to open. - **kwargs - Additional parameters are passed through to :func:`zarr.v2.creation.open_array` or - :func:`zarr.v2.hierarchy.open_group`. - - Returns - ------- - z : :class:`zarr.v2.core.Array` or :class:`zarr.v2.hierarchy.Group` - Array or group, depending on what exists in the given store. - - See Also - -------- - zarr.v2.creation.open_array, zarr.v2.hierarchy.open_group - - Examples - -------- - - Storing data in a directory 'data/example.zarr' on the local file system:: - - >>> import zarr - >>> store = 'data/example.zarr' - >>> zw = zarr.v2.open(store, mode='w', shape=100, dtype='i4') # open new array - >>> zw - - >>> za = zarr.v2.open(store, mode='a') # open existing array for reading and writing - >>> za - - >>> zr = zarr.v2.open(store, mode='r') # open existing array read-only - >>> zr - - >>> gw = zarr.v2.open(store, mode='w') # open new group, overwriting previous data - >>> gw - - >>> ga = zarr.v2.open(store, mode='a') # open existing group for reading and writing - >>> ga - - >>> gr = zarr.v2.open(store, mode='r') # open existing group read-only - >>> gr - - - """ - - # handle polymorphic store arg - # we pass storage options explicitly, since normalize_store_arg might construct - # a store if the input is a fsspec-compatible URL - _store: BaseStore = normalize_store_arg( - store, storage_options=kwargs.pop("storage_options", {}), mode=mode - ) - # path = _check_and_update_path(_store, path) - path = normalize_storage_path(path) - kwargs["path"] = path - - if mode in {"w", "w-", "x"}: - if "shape" in kwargs: - return open_array(_store, mode=mode, **kwargs) - else: - return open_group(_store, mode=mode, **kwargs) - - elif mode == "a": - if "shape" in kwargs or contains_array(_store, path): - return open_array(_store, mode=mode, **kwargs) - else: - return open_group(_store, mode=mode, **kwargs) - - else: - if contains_array(_store, path): - return open_array(_store, mode=mode, **kwargs) - elif contains_group(_store, path): - return open_group(_store, mode=mode, **kwargs) - else: - raise PathNotFoundError(path) - - -def _might_close(path): - return isinstance(path, (str, os.PathLike)) - - -def save_array(store: StoreLike, arr, *, path=None, **kwargs): - """Convenience function to save a NumPy array to the local file system, following a - similar API to the NumPy save() function. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - arr : ndarray - NumPy array with data to save. - path : str or None, optional - The path within the store where the array will be saved. - kwargs - Passed through to :func:`create`, e.g., compressor. - - Examples - -------- - Save an array to a directory on the file system (uses a :class:`DirectoryStore`):: - - >>> import zarr - >>> import numpy as np - >>> arr = np.arange(10000) - >>> zarr.v2.save_array('data/example.zarr', arr) - >>> zarr.v2.load('data/example.zarr') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - Save an array to a single file (uses a :class:`ZipStore`):: - - >>> zarr.v2.save_array('data/example.zip', arr) - >>> zarr.v2.load('data/example.zip') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - """ - may_need_closing = _might_close(store) - _store: BaseStore = normalize_store_arg(store, mode="w") - path = _check_and_update_path(_store, path) - try: - _create_array(arr, store=_store, overwrite=True, path=path, **kwargs) - finally: - if may_need_closing: - # needed to ensure zip file records are written - _store.close() - - -def save_group(store: StoreLike, *args, path=None, **kwargs): - """Convenience function to save several NumPy arrays to the local file system, following a - similar API to the NumPy savez()/savez_compressed() functions. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - args : ndarray - NumPy arrays with data to save. - path : str or None, optional - Path within the store where the group will be saved. - kwargs - NumPy arrays with data to save. - - Examples - -------- - Save several arrays to a directory on the file system (uses a - :class:`DirectoryStore`): - - >>> import zarr - >>> import numpy as np - >>> a1 = np.arange(10000) - >>> a2 = np.arange(10000, 0, -1) - >>> zarr.v2.save_group('data/example.zarr', a1, a2) - >>> loader = zarr.v2.load('data/example.zarr') - >>> loader - - >>> loader['arr_0'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['arr_1'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Save several arrays using named keyword arguments:: - - >>> zarr.v2.save_group('data/example.zarr', foo=a1, bar=a2) - >>> loader = zarr.v2.load('data/example.zarr') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Store several arrays in a single zip file (uses a :class:`ZipStore`):: - - >>> zarr.v2.save_group('data/example.zip', foo=a1, bar=a2) - >>> loader = zarr.v2.load('data/example.zip') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Notes - ----- - Default compression options will be used. - - """ - if len(args) == 0 and len(kwargs) == 0: - raise ValueError("at least one array must be provided") - # handle polymorphic store arg - may_need_closing = _might_close(store) - _store: BaseStore = normalize_store_arg(store, mode="w") - path = _check_and_update_path(_store, path) - try: - grp = _create_group(_store, path=path, overwrite=True) - for i, arr in enumerate(args): - k = "arr_{}".format(i) - grp.create_dataset(k, data=arr, overwrite=True) - for k, arr in kwargs.items(): - grp.create_dataset(k, data=arr, overwrite=True) - finally: - if may_need_closing: - # needed to ensure zip file records are written - _store.close() - - -def save(store: StoreLike, *args, path=None, **kwargs): - """Convenience function to save an array or group of arrays to the local file system. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - args : ndarray - NumPy arrays with data to save. - path : str or None, optional - The path within the group where the arrays will be saved. - kwargs - NumPy arrays with data to save. - - Examples - -------- - Save an array to a directory on the file system (uses a :class:`DirectoryStore`):: - - >>> import zarr - >>> import numpy as np - >>> arr = np.arange(10000) - >>> zarr.v2.save('data/example.zarr', arr) - >>> zarr.v2.load('data/example.zarr') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - Save an array to a Zip file (uses a :class:`ZipStore`):: - - >>> zarr.v2.save('data/example.zip', arr) - >>> zarr.v2.load('data/example.zip') - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - - Save several arrays to a directory on the file system (uses a - :class:`DirectoryStore` and stores arrays in a group):: - - >>> import zarr - >>> import numpy as np - >>> a1 = np.arange(10000) - >>> a2 = np.arange(10000, 0, -1) - >>> zarr.v2.save('data/example.zarr', a1, a2) - >>> loader = zarr.v2.load('data/example.zarr') - >>> loader - - >>> loader['arr_0'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['arr_1'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Save several arrays using named keyword arguments:: - - >>> zarr.v2.save('data/example.zarr', foo=a1, bar=a2) - >>> loader = zarr.v2.load('data/example.zarr') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - Store several arrays in a single zip file (uses a :class:`ZipStore`):: - - >>> zarr.v2.save('data/example.zip', foo=a1, bar=a2) - >>> loader = zarr.v2.load('data/example.zip') - >>> loader - - >>> loader['foo'] - array([ 0, 1, 2, ..., 9997, 9998, 9999]) - >>> loader['bar'] - array([10000, 9999, 9998, ..., 3, 2, 1]) - - See Also - -------- - save_array, save_group - - """ - if len(args) == 0 and len(kwargs) == 0: - raise ValueError("at least one array must be provided") - if len(args) == 1 and len(kwargs) == 0: - save_array(store, args[0], path=path) - else: - save_group(store, *args, path=path, **kwargs) - - -class LazyLoader(Mapping): - def __init__(self, grp): - self.grp = grp - self.cache = dict() - - def __getitem__(self, item): - try: - return self.cache[item] - except KeyError: - arr = self.grp[item][...] - self.cache[item] = arr - return arr - - def __len__(self): - return len(self.grp) - - def __iter__(self): - return iter(self.grp) - - def __contains__(self, item): - return item in self.grp - - def __repr__(self): - r = ">> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('qux') - >>> d1 = g5.create_dataset('baz', shape=100, chunks=10) - >>> g1.tree() - / - ├── bar - │ ├── baz - │ └── qux - │ └── baz (100,) float64 - └── foo - >>> import h5py - >>> h5f = h5py.File('data/example.h5', mode='w') - >>> zarr.v2.copy_all(g1, h5f) - (5, 0, 800) - >>> zarr.v2.tree(h5f) - / - ├── bar - │ ├── baz - │ └── qux - │ └── baz (100,) float64 - └── foo - - See Also - -------- - zarr.v2.hierarchy.Group.tree - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default output and/or parameters may change - in future versions. - - """ - - return TreeViewer(grp, expand=expand, level=level) - - -class _LogWriter: - def __init__(self, log): - self.log_func = None - self.log_file = None - self.needs_closing = False - if log is None: - # don't do any logging - pass - elif callable(log): - self.log_func = log - elif isinstance(log, str): - self.log_file = _builtin_open(log, mode="w") - self.needs_closing = True - elif hasattr(log, "write"): - self.log_file = log - else: - raise TypeError( - "log must be a callable function, file path or file-like object, found %r" % log - ) - - def __enter__(self): - return self - - def __exit__(self, *args): - if self.log_file is not None and self.needs_closing: - self.log_file.close() - - def __call__(self, *args, **kwargs): - if self.log_file is not None: - kwargs["file"] = self.log_file - print(*args, **kwargs) - if hasattr(self.log_file, "flush"): - # get immediate feedback - self.log_file.flush() - elif self.log_func is not None: - self.log_func(*args, **kwargs) - - -def _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied): - # log a final message with a summary of what happened - if dry_run: - message = "dry run: " - else: - message = "all done: " - message += "{:,} copied, {:,} skipped".format(n_copied, n_skipped) - if not dry_run: - message += ", {:,} bytes copied".format(n_bytes_copied) - log(message) - - -def copy_store( - source, - dest, - source_path="", - dest_path="", - excludes=None, - includes=None, - flags=0, - if_exists="raise", - dry_run=False, - log=None, -): - """Copy data directly from the `source` store to the `dest` store. Use this - function when you want to copy a group or array in the most efficient way, - preserving all configuration and attributes. This function is more efficient - than the copy() or copy_all() functions because it avoids de-compressing and - re-compressing data, rather the compressed chunk data for each array are - copied directly between stores. - - Parameters - ---------- - source : Mapping - Store to copy data from. - dest : MutableMapping - Store to copy data into. - source_path : str, optional - Only copy data from under this path in the source store. - dest_path : str, optional - Copy data into this path in the destination store. - excludes : sequence of str, optional - One or more regular expressions which will be matched against keys in - the source store. Any matching key will not be copied. - includes : sequence of str, optional - One or more regular expressions which will be matched against keys in - the source store and will override any excludes also matching. - flags : int, optional - Regular expression flags used for matching excludes and includes. - if_exists : {'raise', 'replace', 'skip'}, optional - How to handle keys that already exist in the destination store. If - 'raise' then a CopyError is raised on the first key already present - in the destination store. If 'replace' then any data will be replaced in - the destination. If 'skip' then any existing keys will not be copied. - dry_run : bool, optional - If True, don't actually copy anything, just log what would have - happened. - log : callable, file path or file-like object, optional - If provided, will be used to log progress information. - - Returns - ------- - n_copied : int - Number of items copied. - n_skipped : int - Number of items skipped. - n_bytes_copied : int - Number of bytes of data that were actually copied. - - Examples - -------- - - >>> import zarr - >>> store1 = zarr.v2.DirectoryStore('data/example.zarr') - >>> root = zarr.v2.group(store1, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.create_group('bar') - >>> baz = bar.create_dataset('baz', shape=100, chunks=50, dtype='i8') - >>> import numpy as np - >>> baz[:] = np.arange(100) - >>> root.tree() - / - └── foo - └── bar - └── baz (100,) int64 - >>> from sys import stdout - >>> store2 = zarr.v2.ZipStore('data/example.zip', mode='w') - >>> zarr.v2.copy_store(store1, store2, log=stdout) - copy .zgroup - copy foo/.zgroup - copy foo/bar/.zgroup - copy foo/bar/baz/.zarray - copy foo/bar/baz/0 - copy foo/bar/baz/1 - all done: 6 copied, 0 skipped, 566 bytes copied - (6, 0, 566) - >>> new_root = zarr.v2.group(store2) - >>> new_root.tree() - / - └── foo - └── bar - └── baz (100,) int64 - >>> new_root['foo/bar/baz'][:] - array([ 0, 1, 2, ..., 97, 98, 99]) - >>> store2.close() # zip stores need to be closed - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default behaviour and/or parameters may change - in future versions. - - """ - - # normalize paths - source_path = normalize_storage_path(source_path) - dest_path = normalize_storage_path(dest_path) - if source_path: - source_path = source_path + "/" - if dest_path: - dest_path = dest_path + "/" - - # normalize excludes and includes - if excludes is None: - excludes = [] - elif isinstance(excludes, str): - excludes = [excludes] - if includes is None: - includes = [] - elif isinstance(includes, str): - includes = [includes] - excludes = [re.compile(e, flags) for e in excludes] - includes = [re.compile(i, flags) for i in includes] - - # check if_exists parameter - valid_if_exists = ["raise", "replace", "skip"] - if if_exists not in valid_if_exists: - raise ValueError( - "if_exists must be one of {!r}; found {!r}".format(valid_if_exists, if_exists) - ) - - # setup counting variables - n_copied = n_skipped = n_bytes_copied = 0 - - source_store_version = getattr(source, "_store_version", 2) - dest_store_version = getattr(dest, "_store_version", 2) - if source_store_version != dest_store_version: - raise ValueError("zarr stores must share the same protocol version") - - if source_store_version > 2: - raise NotImplementedError("This function only supports Zarr version 2.") - - # setup logging - with _LogWriter(log) as log: - # iterate over source keys - for source_key in sorted(source.keys()): - # filter to keys under source path - if source_store_version == 2: - if not source_key.startswith(source_path): - continue - elif source_store_version == 3: - raise NotImplementedError("This function only supports Zarr version 2.") - # process excludes and includes - exclude = False - for prog in excludes: - if prog.search(source_key): - exclude = True - break - if exclude: - for prog in includes: - if prog.search(source_key): - exclude = False - break - if exclude: - continue - - # map key to destination path - if source_store_version == 2: - key_suffix = source_key[len(source_path) :] - dest_key = dest_path + key_suffix - elif source_store_version == 3: - raise NotImplementedError("This function only supports Zarr version 2.") - # create a descriptive label for this operation - descr = source_key - if dest_key != source_key: - descr = descr + " -> " + dest_key - - # decide what to do - do_copy = True - if if_exists != "replace" and dest_key in dest: - if if_exists == "raise": - raise CopyError("key {!r} exists in destination".format(dest_key)) - elif if_exists == "skip": - do_copy = False - - # take action - if do_copy: - log("copy {}".format(descr)) - if not dry_run: - data = source[source_key] - n_bytes_copied += buffer_size(data) - dest[dest_key] = data - n_copied += 1 - else: - log("skip {}".format(descr)) - n_skipped += 1 - - # log a final message with a summary of what happened - _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied) - - return n_copied, n_skipped, n_bytes_copied - - -def _check_dest_is_group(dest): - if not hasattr(dest, "create_dataset"): - raise ValueError("dest must be a group, got {!r}".format(dest)) - - -def copy( - source, - dest, - name=None, - shallow=False, - without_attrs=False, - log=None, - if_exists="raise", - dry_run=False, - **create_kws, -): - """Copy the `source` array or group into the `dest` group. - - Parameters - ---------- - source : group or array/dataset - A zarr group or array, or an h5py group or dataset. - dest : group - A zarr or h5py group. - name : str, optional - Name to copy the object to. - shallow : bool, optional - If True, only copy immediate children of `source`. - without_attrs : bool, optional - Do not copy user attributes. - log : callable, file path or file-like object, optional - If provided, will be used to log progress information. - if_exists : {'raise', 'replace', 'skip', 'skip_initialized'}, optional - How to handle arrays that already exist in the destination group. If - 'raise' then a CopyError is raised on the first array already present - in the destination group. If 'replace' then any array will be - replaced in the destination. If 'skip' then any existing arrays will - not be copied. If 'skip_initialized' then any existing arrays with - all chunks initialized will not be copied (not available when copying to - h5py). - dry_run : bool, optional - If True, don't actually copy anything, just log what would have - happened. - **create_kws - Passed through to the create_dataset method when copying an array/dataset. - - Returns - ------- - n_copied : int - Number of items copied. - n_skipped : int - Number of items skipped. - n_bytes_copied : int - Number of bytes of data that were actually copied. - - Examples - -------- - Here's an example of copying a group named 'foo' from an HDF5 file to a - Zarr group:: - - >>> import h5py - >>> import zarr - >>> import numpy as np - >>> source = h5py.File('data/example.h5', mode='w') - >>> foo = source.create_group('foo') - >>> baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,)) - >>> spam = source.create_dataset('spam', data=np.arange(100, 200), chunks=(30,)) - >>> zarr.v2.tree(source) - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> dest = zarr.v2.group() - >>> from sys import stdout - >>> zarr.v2.copy(source['foo'], dest, log=stdout) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - all done: 3 copied, 0 skipped, 800 bytes copied - (3, 0, 800) - >>> dest.tree() # N.B., no spam - / - └── foo - └── bar - └── baz (100,) int64 - >>> source.close() - - The ``if_exists`` parameter provides options for how to handle pre-existing data in - the destination. Here are some examples of these options, also using - ``dry_run=True`` to find out what would happen without actually copying anything:: - - >>> source = zarr.v2.group() - >>> dest = zarr.v2.group() - >>> baz = source.create_dataset('foo/bar/baz', data=np.arange(100)) - >>> spam = source.create_dataset('foo/spam', data=np.arange(1000)) - >>> existing_spam = dest.create_dataset('foo/spam', data=np.arange(1000)) - >>> from sys import stdout - >>> try: - ... zarr.v2.copy(source['foo'], dest, log=stdout, dry_run=True) - ... except zarr.v2.CopyError as e: - ... print(e) - ... - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - an object 'spam' already exists in destination '/foo' - >>> zarr.v2.copy(source['foo'], dest, log=stdout, if_exists='replace', dry_run=True) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - copy /foo/spam (1000,) int64 - dry run: 4 copied, 0 skipped - (4, 0, 0) - >>> zarr.v2.copy(source['foo'], dest, log=stdout, if_exists='skip', dry_run=True) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - skip /foo/spam (1000,) int64 - dry run: 3 copied, 1 skipped - (3, 1, 0) - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default behaviour and/or parameters may change - in future versions. - - """ - - # value checks - _check_dest_is_group(dest) - - # setup logging - with _LogWriter(log) as log: - # do the copying - n_copied, n_skipped, n_bytes_copied = _copy( - log, - source, - dest, - name=name, - root=True, - shallow=shallow, - without_attrs=without_attrs, - if_exists=if_exists, - dry_run=dry_run, - **create_kws, - ) - - # log a final message with a summary of what happened - _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied) - - return n_copied, n_skipped, n_bytes_copied - - -def _copy(log, source, dest, name, root, shallow, without_attrs, if_exists, dry_run, **create_kws): - # N.B., if this is a dry run, dest may be None - - # setup counting variables - n_copied = n_skipped = n_bytes_copied = 0 - - # are we copying to/from h5py? - source_h5py = source.__module__.startswith("h5py.") - dest_h5py = dest is not None and dest.__module__.startswith("h5py.") - - # check if_exists parameter - valid_if_exists = ["raise", "replace", "skip", "skip_initialized"] - if if_exists not in valid_if_exists: - raise ValueError( - "if_exists must be one of {!r}; found {!r}".format(valid_if_exists, if_exists) - ) - if dest_h5py and if_exists == "skip_initialized": - raise ValueError("{!r} can only be used when copying to zarr".format(if_exists)) - - # determine name to copy to - if name is None: - name = source.name.split("/")[-1] - if not name: - # this can happen if source is the root group - raise TypeError( - "source has no name, please provide the `name` " - "parameter to indicate a name to copy to" - ) - - if hasattr(source, "shape"): - # copy a dataset/array - - # check if already exists, decide what to do - do_copy = True - exists = dest is not None and name in dest - if exists: - if if_exists == "raise": - raise CopyError( - "an object {!r} already exists in destination {!r}".format(name, dest.name) - ) - elif if_exists == "skip": - do_copy = False - elif if_exists == "skip_initialized": - ds = dest[name] - if ds.nchunks_initialized == ds.nchunks: - do_copy = False - - # take action - if do_copy: - # log a message about what we're going to do - log("copy {} {} {}".format(source.name, source.shape, source.dtype)) - - if not dry_run: - # clear the way - if exists: - del dest[name] - - # setup creation keyword arguments - kws = create_kws.copy() - - # setup chunks option, preserve by default - kws.setdefault("chunks", source.chunks) - - # setup compression options - if source_h5py: - if dest_h5py: - # h5py -> h5py; preserve compression options by default - kws.setdefault("compression", source.compression) - kws.setdefault("compression_opts", source.compression_opts) - kws.setdefault("shuffle", source.shuffle) - kws.setdefault("fletcher32", source.fletcher32) - kws.setdefault("fillvalue", source.fillvalue) - else: - # h5py -> zarr; use zarr default compression options - kws.setdefault("fill_value", source.fillvalue) - else: - if dest_h5py: - # zarr -> h5py; use some vaguely sensible defaults - kws.setdefault("chunks", True) - kws.setdefault("compression", "gzip") - kws.setdefault("compression_opts", 1) - kws.setdefault("shuffle", False) - kws.setdefault("fillvalue", source.fill_value) - else: - # zarr -> zarr; preserve compression options by default - kws.setdefault("compressor", source.compressor) - kws.setdefault("filters", source.filters) - kws.setdefault("order", source.order) - kws.setdefault("fill_value", source.fill_value) - - # create new dataset in destination - ds = dest.create_dataset(name, shape=source.shape, dtype=source.dtype, **kws) - - # copy data - N.B., go chunk by chunk to avoid loading - # everything into memory - shape = ds.shape - chunks = ds.chunks - chunk_offsets = [range(0, s, c) for s, c in zip(shape, chunks)] - for offset in itertools.product(*chunk_offsets): - sel = tuple(slice(o, min(s, o + c)) for o, s, c in zip(offset, shape, chunks)) - ds[sel] = source[sel] - n_bytes_copied += ds.size * ds.dtype.itemsize - - # copy attributes - if not without_attrs: - if dest_h5py and "filters" in source.attrs: - # No filters key in v3 metadata so it was stored in the - # attributes instead. We cannot copy this key to - # HDF5 attrs, though! - source_attrs = source.attrs.asdict().copy() - source_attrs.pop("filters", None) - else: - source_attrs = source.attrs - ds.attrs.update(source_attrs) - - n_copied += 1 - - else: - log("skip {} {} {}".format(source.name, source.shape, source.dtype)) - n_skipped += 1 - - elif root or not shallow: - # copy a group - - # check if an array is in the way - do_copy = True - exists_array = dest is not None and name in dest and hasattr(dest[name], "shape") - if exists_array: - if if_exists == "raise": - raise CopyError( - "an array {!r} already exists in destination {!r}".format(name, dest.name) - ) - elif if_exists == "skip": - do_copy = False - - # take action - if do_copy: - # log action - log("copy {}".format(source.name)) - - if not dry_run: - # clear the way - if exists_array: - del dest[name] - - # require group in destination - grp = dest.require_group(name) - - # copy attributes - if not without_attrs: - grp.attrs.update(source.attrs) - - else: - # setup for dry run without creating any groups in the - # destination - if dest is not None: - grp = dest.get(name, None) - else: - grp = None - - # recurse - for k in source.keys(): - c, s, b = _copy( - log, - source[k], - grp, - name=k, - root=False, - shallow=shallow, - without_attrs=without_attrs, - if_exists=if_exists, - dry_run=dry_run, - **create_kws, - ) - n_copied += c - n_skipped += s - n_bytes_copied += b - - n_copied += 1 - - else: - log("skip {}".format(source.name)) - n_skipped += 1 - - return n_copied, n_skipped, n_bytes_copied - - -def copy_all( - source, - dest, - shallow=False, - without_attrs=False, - log=None, - if_exists="raise", - dry_run=False, - **create_kws, -): - """Copy all children of the `source` group into the `dest` group. - - Parameters - ---------- - source : group or array/dataset - A zarr group or array, or an h5py group or dataset. - dest : group - A zarr or h5py group. - shallow : bool, optional - If True, only copy immediate children of `source`. - without_attrs : bool, optional - Do not copy user attributes. - log : callable, file path or file-like object, optional - If provided, will be used to log progress information. - if_exists : {'raise', 'replace', 'skip', 'skip_initialized'}, optional - How to handle arrays that already exist in the destination group. If - 'raise' then a CopyError is raised on the first array already present - in the destination group. If 'replace' then any array will be - replaced in the destination. If 'skip' then any existing arrays will - not be copied. If 'skip_initialized' then any existing arrays with - all chunks initialized will not be copied (not available when copying to - h5py). - dry_run : bool, optional - If True, don't actually copy anything, just log what would have - happened. - **create_kws - Passed through to the create_dataset method when copying an - array/dataset. - - Returns - ------- - n_copied : int - Number of items copied. - n_skipped : int - Number of items skipped. - n_bytes_copied : int - Number of bytes of data that were actually copied. - - Examples - -------- - >>> import h5py - >>> import zarr - >>> import numpy as np - >>> source = h5py.File('data/example.h5', mode='w') - >>> foo = source.create_group('foo') - >>> baz = foo.create_dataset('bar/baz', data=np.arange(100), chunks=(50,)) - >>> spam = source.create_dataset('spam', data=np.arange(100, 200), chunks=(30,)) - >>> zarr.v2.tree(source) - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> dest = zarr.v2.group() - >>> import sys - >>> zarr.v2.copy_all(source, dest, log=sys.stdout) - copy /foo - copy /foo/bar - copy /foo/bar/baz (100,) int64 - copy /spam (100,) int64 - all done: 4 copied, 0 skipped, 1,600 bytes copied - (4, 0, 1600) - >>> dest.tree() - / - ├── foo - │ └── bar - │ └── baz (100,) int64 - └── spam (100,) int64 - >>> source.close() - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default behaviour and/or parameters may change - in future versions. - - """ - - # value checks - _check_dest_is_group(dest) - - # setup counting variables - n_copied = n_skipped = n_bytes_copied = 0 - - # setup logging - with _LogWriter(log) as log: - for k in source.keys(): - c, s, b = _copy( - log, - source[k], - dest, - name=k, - root=False, - shallow=shallow, - without_attrs=without_attrs, - if_exists=if_exists, - dry_run=dry_run, - **create_kws, - ) - n_copied += c - n_skipped += s - n_bytes_copied += b - - dest.attrs.update(**source.attrs) - - # log a final message with a summary of what happened - _log_copy_summary(log, dry_run, n_copied, n_skipped, n_bytes_copied) - - return n_copied, n_skipped, n_bytes_copied - - -def consolidate_metadata(store: BaseStore, metadata_key=".zmetadata", *, path=""): - """ - Consolidate all metadata for groups and arrays within the given store - into a single resource and put it under the given key. - - This produces a single object in the backend store, containing all the - metadata read from all the zarr-related keys that can be found. After - metadata have been consolidated, use :func:`open_consolidated` to open - the root group in optimised, read-only mode, using the consolidated - metadata to reduce the number of read operations on the backend store. - - Note, that if the metadata in the store is changed after this - consolidation, then the metadata read by :func:`open_consolidated` - would be incorrect unless this function is called again. - - .. note:: This is an experimental feature. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - metadata_key : str - Key to put the consolidated metadata under. - path : str or None - Path corresponding to the group that is being consolidated. Not required - for zarr v2 stores. - - Returns - ------- - g : :class:`zarr.v2.hierarchy.Group` - Group instance, opened with the new consolidated metadata. - - See Also - -------- - open_consolidated - - """ - store = normalize_store_arg(store, mode="w") - - version = store._store_version - - if version == 2: - - def is_zarr_key(key): - return key.endswith(".zarray") or key.endswith(".zgroup") or key.endswith(".zattrs") - - else: - raise NotImplementedError("This function only supports Zarr version 2.") - out = { - "zarr_consolidated_format": 1, - "metadata": {key: json_loads(store[key]) for key in store if is_zarr_key(key)}, - } - store[metadata_key] = json_dumps(out) - return open_consolidated(store, metadata_key=metadata_key, path=path) - - -def open_consolidated(store: StoreLike, metadata_key=".zmetadata", mode="r+", **kwargs): - """Open group using metadata previously consolidated into a single key. - - This is an optimised method for opening a Zarr group, where instead of - traversing the group/array hierarchy by accessing the metadata keys at - each level, a single key contains all of the metadata for everything. - For remote data sources where the overhead of accessing a key is large - compared to the time to read data. - - The group accessed must have already had its metadata consolidated into a - single key using the function :func:`consolidate_metadata`. - - This optimised method only works in modes which do not change the - metadata, although the data may still be written/updated. - - Parameters - ---------- - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - metadata_key : str - Key to read the consolidated metadata from. The default (.zmetadata) - corresponds to the default used by :func:`consolidate_metadata`. - mode : {'r', 'r+'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist) although only writes to data are allowed, - changes to metadata including creation of new arrays or group - are not allowed. - **kwargs - Additional parameters are passed through to :func:`zarr.v2.creation.open_array` or - :func:`zarr.v2.hierarchy.open_group`. - - Returns - ------- - g : :class:`zarr.v2.hierarchy.Group` - Group instance, opened with the consolidated metadata. - - See Also - -------- - consolidate_metadata - - """ - - # normalize parameters - store = normalize_store_arg(store, storage_options=kwargs.get("storage_options"), mode=mode) - if mode not in {"r", "r+"}: - raise ValueError("invalid mode, expected either 'r' or 'r+'; found {!r}".format(mode)) - - path = kwargs.pop("path", None) - if store._store_version == 2: - ConsolidatedStoreClass = ConsolidatedMetadataStore - else: - raise NotImplementedError("This function only supports Zarr version 2.") - - # setup metadata store - meta_store = ConsolidatedStoreClass(store, metadata_key=metadata_key) - - # pass through - chunk_store = kwargs.pop("chunk_store", None) or store - return open(store=meta_store, chunk_store=chunk_store, mode=mode, path=path, **kwargs) diff --git a/src/zarr/v2/core.py b/src/zarr/v2/core.py deleted file mode 100644 index 9eeb467d68..0000000000 --- a/src/zarr/v2/core.py +++ /dev/null @@ -1,2855 +0,0 @@ -import binascii -import hashlib -import itertools -import math -import operator -import re -from functools import reduce -from typing import Any - -import numpy as np -from numcodecs import AsType, get_codec -from numcodecs.compat import ensure_bytes, ensure_ndarray_like - -from zarr.v2._storage.store import _prefix_to_attrs_key -from zarr.v2.attrs import Attributes -from zarr.v2.context import Context -from zarr.v2.errors import ArrayNotFoundError, ReadOnlyError, ArrayIndexError -from zarr.v2.indexing import ( - BasicIndexer, - CoordinateIndexer, - MaskIndexer, - OIndex, - OrthogonalIndexer, - VIndex, - BlockIndex, - BlockIndexer, - PartialChunkIterator, - check_fields, - check_no_multi_fields, - ensure_tuple, - err_too_many_indices, - is_contiguous_selection, - is_pure_fancy_indexing, - is_pure_orthogonal_indexing, - is_scalar, - pop_fields, -) -from zarr.v2.storage import ( - _prefix_to_array_key, - KVStore, - getsize, - listdir, - normalize_store_arg, -) -from zarr.v2.util import ( - ConstantMap, - UncompressedPartialReadBufferV3, - all_equal, - InfoReporter, - check_array_shape, - human_readable_size, - is_total_slice, - nolock, - normalize_chunks, - normalize_resize_args, - normalize_shape, - normalize_storage_path, - PartialReadBuffer, -) - -__all__ = ["Array"] - - -# noinspection PyUnresolvedReferences -class Array: - """Instantiate an array from an initialized store. - - Parameters - ---------- - store : MutableMapping - Array store, already initialized. - path : string, optional - Storage path. - read_only : bool, optional - True if array should be protected against modification. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - synchronizer : object, optional - Array synchronizer. - cache_metadata : bool, optional - If True (default), array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - partial_decompress : bool, optional - If True and while the chunk_store is a FSStore and the compression used - is Blosc, when getting data from the array chunks will be partially - read and decompressed when possible. - - .. versionadded:: 2.7 - - write_empty_chunks : bool, optional - If True, all chunks will be stored regardless of their contents. If - False (default), each chunk is compared to the array's fill value prior - to storing. If a chunk is uniformly equal to the fill value, then that - chunk is not be stored, and the store entry for that chunk's key is - deleted. This setting enables sparser storage, as only chunks with - non-fill-value data are stored, at the expense of overhead associated - with checking the data of each chunk. - - .. versionadded:: 2.11 - - meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. - - .. versionadded:: 2.13 - """ - - def __init__( - self, - store: Any, # BaseStore not strictly required due to normalize_store_arg - path=None, - read_only=False, - chunk_store=None, - synchronizer=None, - cache_metadata=True, - cache_attrs=True, - partial_decompress=False, - write_empty_chunks=True, - meta_array=None, - ): - # N.B., expect at this point store is fully initialized with all - # configuration metadata fully specified and normalized - store = normalize_store_arg(store) - - if chunk_store is not None: - chunk_store = normalize_store_arg(chunk_store) - - self._store = store - self._chunk_store = chunk_store - self._transformed_chunk_store = None - self._path = normalize_storage_path(path) - if self._path: - self._key_prefix = self._path + "/" - else: - self._key_prefix = "" - self._read_only = bool(read_only) - self._synchronizer = synchronizer - self._cache_metadata = cache_metadata - self._is_view = False - self._partial_decompress = partial_decompress - self._write_empty_chunks = write_empty_chunks - if meta_array is not None: - self._meta_array = np.empty_like(meta_array, shape=()) - else: - self._meta_array = np.empty(()) - - # initialize metadata - self._load_metadata() - - # initialize attributes - akey = _prefix_to_attrs_key(self._store, self._key_prefix) - self._attrs = Attributes( - store, key=akey, read_only=read_only, synchronizer=synchronizer, cache=cache_attrs - ) - - # initialize info reporter - self._info_reporter = InfoReporter(self) - - # initialize indexing helpers - self._oindex = OIndex(self) - self._vindex = VIndex(self) - self._blocks = BlockIndex(self) - - def _load_metadata(self): - """(Re)load metadata from store.""" - if self._synchronizer is None: - self._load_metadata_nosync() - else: - mkey = _prefix_to_array_key(self._store, self._key_prefix) - with self._synchronizer[mkey]: - self._load_metadata_nosync() - - def _load_metadata_nosync(self): - try: - mkey = _prefix_to_array_key(self._store, self._key_prefix) - meta_bytes = self._store[mkey] - except KeyError: - raise ArrayNotFoundError(self._path) - else: - # decode and store metadata as instance members - meta = self._store._metadata_class.decode_array_metadata(meta_bytes) - self._meta = meta - self._shape = meta["shape"] - self._fill_value = meta["fill_value"] - dimension_separator = meta.get("dimension_separator", None) - - self._chunks = meta["chunks"] - self._dtype = meta["dtype"] - self._order = meta["order"] - if dimension_separator is None: - try: - dimension_separator = self._store._dimension_separator - except (AttributeError, KeyError): - pass - - # Fallback for any stores which do not choose a default - if dimension_separator is None: - dimension_separator = "." - - self._dimension_separator = dimension_separator - - # setup compressor - compressor = meta.get("compressor", None) - if compressor is None: - self._compressor = None - else: - self._compressor = get_codec(compressor) - - # setup filters - - filters = meta.get("filters", []) - - if filters: - filters = [get_codec(config) for config in filters] - self._filters = filters - - def _refresh_metadata(self): - if not self._cache_metadata: - self._load_metadata() - - def _refresh_metadata_nosync(self): - if not self._cache_metadata and not self._is_view: - self._load_metadata_nosync() - - def _flush_metadata_nosync(self): - if self._is_view: - raise PermissionError("operation not permitted for views") - - if self._compressor: - compressor_config = self._compressor.get_config() - else: - compressor_config = None - if self._filters: - filters_config = [f.get_config() for f in self._filters] - else: - filters_config = None - _compressor = compressor_config - meta = dict( - shape=self._shape, - compressor=_compressor, - fill_value=self._fill_value, - filters=filters_config, - ) - - meta.update( - dict( - chunks=self._chunks, - dtype=self._dtype, - order=self._order, - dimension_separator=self._dimension_separator, - ) - ) - mkey = _prefix_to_array_key(self._store, self._key_prefix) - self._store[mkey] = self._store._metadata_class.encode_array_metadata(meta) - - @property - def store(self): - """A MutableMapping providing the underlying storage for the array.""" - return self._store - - @property - def path(self): - """Storage path.""" - return self._path - - @property - def name(self): - """Array name following h5py convention.""" - if self.path: - # follow h5py convention: add leading slash - name = self.path - if name[0] != "/": - name = "/" + name - return name - return None - - @property - def basename(self): - """Final component of name.""" - if self.name is not None: - return self.name.split("/")[-1] - return None - - @property - def read_only(self): - """A boolean, True if modification operations are not permitted.""" - return self._read_only - - @read_only.setter - def read_only(self, value): - self._read_only = bool(value) - - @property - def chunk_store(self): - """A MutableMapping providing the underlying storage for array chunks.""" - if self._transformed_chunk_store is not None: - return self._transformed_chunk_store - elif self._chunk_store is not None: - return self._chunk_store - else: - return self._store - - @property - def shape(self): - """A tuple of integers describing the length of each dimension of - the array.""" - # N.B., shape may change if array is resized, hence need to refresh - # metadata - self._refresh_metadata() - return self._shape - - @shape.setter - def shape(self, value): - self.resize(value) - - @property - def chunks(self): - """A tuple of integers describing the length of each dimension of a - chunk of the array.""" - return self._chunks - - @property - def dtype(self): - """The NumPy data type.""" - return self._dtype - - @property - def compressor(self): - """Primary compression codec.""" - return self._compressor - - @property - def fill_value(self): - """A value used for uninitialized portions of the array.""" - return self._fill_value - - @fill_value.setter - def fill_value(self, new): - self._fill_value = new - self._flush_metadata_nosync() - - @property - def order(self): - """A string indicating the order in which bytes are arranged within - chunks of the array.""" - return self._order - - @property - def filters(self): - """One or more codecs used to transform data prior to compression.""" - return self._filters - - @property - def synchronizer(self): - """Object used to synchronize write access to the array.""" - return self._synchronizer - - @property - def attrs(self): - """A MutableMapping containing user-defined attributes. Note that - attribute values must be JSON serializable.""" - return self._attrs - - @property - def ndim(self): - """Number of dimensions.""" - return len(self._shape) - - @property - def _size(self): - return reduce(operator.mul, self._shape, 1) - - @property - def size(self): - """The total number of elements in the array.""" - # N.B., this property depends on shape, and shape may change if array - # is resized, hence need to refresh metadata - self._refresh_metadata() - return self._size - - @property - def itemsize(self): - """The size in bytes of each item in the array.""" - return self.dtype.itemsize - - @property - def _nbytes(self): - return self._size * self.itemsize - - @property - def nbytes(self): - """The total number of bytes that would be required to store the - array without compression.""" - # N.B., this property depends on shape, and shape may change if array - # is resized, hence need to refresh metadata - self._refresh_metadata() - return self._nbytes - - @property - def nbytes_stored(self): - """The total number of stored bytes of data for the array. This - includes storage required for configuration metadata and user - attributes.""" - m = getsize(self._store, self._path) - if self._chunk_store is None: - return m - else: - n = getsize(self._chunk_store, self._path) - if m < 0 or n < 0: - return -1 - else: - return m + n - - @property - def _cdata_shape(self): - if self._shape == (): - return (1,) - else: - return tuple(math.ceil(s / c) for s, c in zip(self._shape, self._chunks)) - - @property - def cdata_shape(self): - """A tuple of integers describing the number of chunks along each - dimension of the array.""" - self._refresh_metadata() - return self._cdata_shape - - @property - def _nchunks(self): - return reduce(operator.mul, self._cdata_shape, 1) - - @property - def nchunks(self): - """Total number of chunks.""" - self._refresh_metadata() - return self._nchunks - - @property - def nchunks_initialized(self): - """The number of chunks that have been initialized with some data.""" - - # key pattern for chunk keys - prog = re.compile(r"\.".join([r"\d+"] * min(1, self.ndim))) - - # count chunk keys - return sum(1 for k in listdir(self.chunk_store, self._path) if prog.match(k)) - - # backwards compatibility - initialized = nchunks_initialized - - @property - def is_view(self): - """A boolean, True if this array is a view on another array.""" - return self._is_view - - @property - def oindex(self): - """Shortcut for orthogonal (outer) indexing, see :func:`get_orthogonal_selection` and - :func:`set_orthogonal_selection` for documentation and examples.""" - return self._oindex - - @property - def vindex(self): - """Shortcut for vectorized (inner) indexing, see :func:`get_coordinate_selection`, - :func:`set_coordinate_selection`, :func:`get_mask_selection` and - :func:`set_mask_selection` for documentation and examples.""" - return self._vindex - - @property - def blocks(self): - """Shortcut for blocked chunked indexing, see :func:`get_block_selection` and - :func:`set_block_selection` for documentation and examples.""" - return self._blocks - - @property - def write_empty_chunks(self) -> bool: - """A Boolean, True if chunks composed of the array's fill value - will be stored. If False, such chunks will not be stored. - """ - return self._write_empty_chunks - - @property - def meta_array(self): - """An array-like instance to use for determining arrays to create and return - to users. - """ - return self._meta_array - - def __eq__(self, other): - return ( - isinstance(other, Array) - and self.store == other.store - and self.read_only == other.read_only - and self.path == other.path - and not self._is_view - # N.B., no need to compare other properties, should be covered by - # store comparison - ) - - def __array__(self, *args): - a = self[...] - if args: - a = a.astype(args[0]) - return a - - def islice(self, start=None, end=None): - """ - Yield a generator for iterating over the entire or parts of the - array. Uses a cache so chunks only have to be decompressed once. - - Parameters - ---------- - start : int, optional - Start index for the generator to start at. Defaults to 0. - end : int, optional - End index for the generator to stop at. Defaults to self.shape[0]. - - Yields - ------ - out : generator - A generator that can be used to iterate over the requested region - the array. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.array(np.arange(100)) - - Iterate over part of the array: - >>> for value in z.islice(25, 30): value; - 25 - 26 - 27 - 28 - 29 - """ - - if len(self.shape) == 0: - # Same error as numpy - raise TypeError("iteration over a 0-d array") - if start is None: - start = 0 - if end is None or end > self.shape[0]: - end = self.shape[0] - - if not isinstance(start, int) or start < 0: - raise ValueError("start must be a nonnegative integer") - - if not isinstance(end, int) or end < 0: - raise ValueError("end must be a nonnegative integer") - - # Avoid repeatedly decompressing chunks by iterating over the chunks - # in the first dimension. - chunk_size = self.chunks[0] - chunk = None - for j in range(start, end): - if j % chunk_size == 0: - chunk = self[j : j + chunk_size] - # init chunk if we start offset of chunk borders - elif chunk is None: - chunk_start = j - j % chunk_size - chunk_end = chunk_start + chunk_size - chunk = self[chunk_start:chunk_end] - yield chunk[j % chunk_size] - - def __iter__(self): - return self.islice() - - def __len__(self): - if self.shape: - return self.shape[0] - else: - # 0-dimensional array, same error message as numpy - raise TypeError("len() of unsized object") - - def __getitem__(self, selection): - """Retrieve data for an item or region of the array. - - Parameters - ---------- - selection : tuple - An integer index or slice or tuple of int/slice objects specifying the - requested item or region for each dimension of the array. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested region. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.array(np.arange(100)) - - Retrieve a single item:: - - >>> z[5] - 5 - - Retrieve a region via slicing:: - - >>> z[:5] - array([0, 1, 2, 3, 4]) - >>> z[-5:] - array([95, 96, 97, 98, 99]) - >>> z[5:10] - array([5, 6, 7, 8, 9]) - >>> z[5:10:2] - array([5, 7, 9]) - >>> z[::2] - array([ 0, 2, 4, ..., 94, 96, 98]) - - Load the entire array into memory:: - - >>> z[...] - array([ 0, 1, 2, ..., 97, 98, 99]) - - Setup a 2-dimensional array:: - - >>> z = zarr.v2.array(np.arange(100).reshape(10, 10)) - - Retrieve an item:: - - >>> z[2, 2] - 22 - - Retrieve a region via slicing:: - - >>> z[1:3, 1:3] - array([[11, 12], - [21, 22]]) - >>> z[1:3, :] - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]]) - >>> z[:, 1:3] - array([[ 1, 2], - [11, 12], - [21, 22], - [31, 32], - [41, 42], - [51, 52], - [61, 62], - [71, 72], - [81, 82], - [91, 92]]) - >>> z[0:5:2, 0:5:2] - array([[ 0, 2, 4], - [20, 22, 24], - [40, 42, 44]]) - >>> z[::2, ::2] - array([[ 0, 2, 4, 6, 8], - [20, 22, 24, 26, 28], - [40, 42, 44, 46, 48], - [60, 62, 64, 66, 68], - [80, 82, 84, 86, 88]]) - - Load the entire array into memory:: - - >>> z[...] - array([[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9], - [10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [20, 21, 22, 23, 24, 25, 26, 27, 28, 29], - [30, 31, 32, 33, 34, 35, 36, 37, 38, 39], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], - [50, 51, 52, 53, 54, 55, 56, 57, 58, 59], - [60, 61, 62, 63, 64, 65, 66, 67, 68, 69], - [70, 71, 72, 73, 74, 75, 76, 77, 78, 79], - [80, 81, 82, 83, 84, 85, 86, 87, 88, 89], - [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]) - - For arrays with a structured dtype, specific fields can be retrieved, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.v2.array(a) - >>> z['foo'] - array([b'aaa', b'bbb', b'ccc'], - dtype='|S3') - - Notes - ----- - Slices with step > 1 are supported, but slices with negative step are not. - - Currently the implementation for __getitem__ is provided by - :func:`vindex` if the indexing is pure fancy indexing (ie a - broadcast-compatible tuple of integer array indices), or by - :func:`set_basic_selection` otherwise. - - Effectively, this means that the following indexing modes are supported: - - - integer indexing - - slice indexing - - mixed slice and integer indexing - - boolean indexing - - fancy indexing (vectorized list of integers) - - For specific indexing options including outer indexing, see the - methods listed under See Also. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - set_orthogonal_selection, get_block_selection, set_block_selection, - vindex, oindex, blocks, __setitem__ - - """ - fields, pure_selection = pop_fields(selection) - if is_pure_fancy_indexing(pure_selection, self.ndim): - result = self.vindex[selection] - elif is_pure_orthogonal_indexing(pure_selection, self.ndim): - result = self.get_orthogonal_selection(pure_selection, fields=fields) - else: - result = self.get_basic_selection(pure_selection, fields=fields) - return result - - def get_basic_selection(self, selection=Ellipsis, out=None, fields=None): - """Retrieve data for an item or region of the array. - - Parameters - ---------- - selection : tuple - A tuple specifying the requested item or region for each dimension of the - array. May be any combination of int and/or slice for multidimensional arrays. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested region. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.array(np.arange(100)) - - Retrieve a single item:: - - >>> z.get_basic_selection(5) - 5 - - Retrieve a region via slicing:: - - >>> z.get_basic_selection(slice(5)) - array([0, 1, 2, 3, 4]) - >>> z.get_basic_selection(slice(-5, None)) - array([95, 96, 97, 98, 99]) - >>> z.get_basic_selection(slice(5, 10)) - array([5, 6, 7, 8, 9]) - >>> z.get_basic_selection(slice(5, 10, 2)) - array([5, 7, 9]) - >>> z.get_basic_selection(slice(None, None, 2)) - array([ 0, 2, 4, ..., 94, 96, 98]) - - Setup a 2-dimensional array:: - - >>> z = zarr.v2.array(np.arange(100).reshape(10, 10)) - - Retrieve an item:: - - >>> z.get_basic_selection((2, 2)) - 22 - - Retrieve a region via slicing:: - - >>> z.get_basic_selection((slice(1, 3), slice(1, 3))) - array([[11, 12], - [21, 22]]) - >>> z.get_basic_selection((slice(1, 3), slice(None))) - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [20, 21, 22, 23, 24, 25, 26, 27, 28, 29]]) - >>> z.get_basic_selection((slice(None), slice(1, 3))) - array([[ 1, 2], - [11, 12], - [21, 22], - [31, 32], - [41, 42], - [51, 52], - [61, 62], - [71, 72], - [81, 82], - [91, 92]]) - >>> z.get_basic_selection((slice(0, 5, 2), slice(0, 5, 2))) - array([[ 0, 2, 4], - [20, 22, 24], - [40, 42, 44]]) - >>> z.get_basic_selection((slice(None, None, 2), slice(None, None, 2))) - array([[ 0, 2, 4, 6, 8], - [20, 22, 24, 26, 28], - [40, 42, 44, 46, 48], - [60, 62, 64, 66, 68], - [80, 82, 84, 86, 88]]) - - For arrays with a structured dtype, specific fields can be retrieved, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.v2.array(a) - >>> z.get_basic_selection(slice(2), fields='foo') - array([b'aaa', b'bbb'], - dtype='|S3') - - Notes - ----- - Slices with step > 1 are supported, but slices with negative step are not. - - Currently this method provides the implementation for accessing data via the - square bracket notation (__getitem__). See :func:`__getitem__` for examples - using the alternative notation. - - See Also - -------- - set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - set_orthogonal_selection, get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # handle zero-dimensional arrays - if self._shape == (): - return self._get_basic_selection_zd(selection=selection, out=out, fields=fields) - else: - return self._get_basic_selection_nd(selection=selection, out=out, fields=fields) - - def _get_basic_selection_zd(self, selection, out=None, fields=None): - # special case basic selection for zero-dimensional array - - # check selection is valid - selection = ensure_tuple(selection) - if selection not in ((), (Ellipsis,)): - err_too_many_indices(selection, ()) - - try: - # obtain encoded data for chunk - ckey = self._chunk_key((0,)) - cdata = self.chunk_store[ckey] - - except KeyError: - # chunk not initialized - chunk = np.zeros_like(self._meta_array, shape=(), dtype=self._dtype) - if self._fill_value is not None: - chunk.fill(self._fill_value) - - else: - chunk = self._decode_chunk(cdata) - - # handle fields - if fields: - chunk = chunk[fields] - - # handle selection of the scalar value via empty tuple - if out is None: - out = chunk[selection] - else: - out[selection] = chunk[selection] - - return out - - def _get_basic_selection_nd(self, selection, out=None, fields=None): - # implementation of basic selection for array with at least one dimension - - # setup indexer - indexer = BasicIndexer(selection, self) - - return self._get_selection(indexer=indexer, out=out, fields=fields) - - def get_orthogonal_selection(self, selection, out=None, fields=None): - """Retrieve data by making a selection for each dimension of the array. For - example, if an array has 2 dimensions, allows selecting specific rows and/or - columns. The selection for each dimension can be either an integer (indexing a - single item), a slice, an array of integers, or a Boolean array where True - values indicate a selection. - - Parameters - ---------- - selection : tuple - A selection for each dimension of the array. May be any combination of int, - slice, integer array or Boolean array. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested selection. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.array(np.arange(100).reshape(10, 10)) - - Retrieve rows and columns via any combination of int, slice, integer array and/or - Boolean array:: - - >>> z.get_orthogonal_selection(([1, 4], slice(None))) - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) - >>> z.get_orthogonal_selection((slice(None), [1, 4])) - array([[ 1, 4], - [11, 14], - [21, 24], - [31, 34], - [41, 44], - [51, 54], - [61, 64], - [71, 74], - [81, 84], - [91, 94]]) - >>> z.get_orthogonal_selection(([1, 4], [1, 4])) - array([[11, 14], - [41, 44]]) - >>> sel = np.zeros(z.shape[0], dtype=bool) - >>> sel[1] = True - >>> sel[4] = True - >>> z.get_orthogonal_selection((sel, sel)) - array([[11, 14], - [41, 44]]) - - For convenience, the orthogonal selection functionality is also available via the - `oindex` property, e.g.:: - - >>> z.oindex[[1, 4], :] - array([[10, 11, 12, 13, 14, 15, 16, 17, 18, 19], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]]) - >>> z.oindex[:, [1, 4]] - array([[ 1, 4], - [11, 14], - [21, 24], - [31, 34], - [41, 44], - [51, 54], - [61, 64], - [71, 74], - [81, 84], - [91, 94]]) - >>> z.oindex[[1, 4], [1, 4]] - array([[11, 14], - [41, 44]]) - >>> sel = np.zeros(z.shape[0], dtype=bool) - >>> sel[1] = True - >>> sel[4] = True - >>> z.oindex[sel, sel] - array([[11, 14], - [41, 44]]) - - Notes - ----- - Orthogonal indexing is also known as outer indexing. - - Slices with step > 1 are supported, but slices with negative step are not. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, set_orthogonal_selection, - get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # setup indexer - indexer = OrthogonalIndexer(selection, self) - - return self._get_selection(indexer=indexer, out=out, fields=fields) - - def get_coordinate_selection(self, selection, out=None, fields=None): - """Retrieve a selection of individual items, by providing the indices - (coordinates) for each selected item. - - Parameters - ---------- - selection : tuple - An integer (coordinate) array for each dimension of the array. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested selection. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.array(np.arange(100).reshape(10, 10)) - - Retrieve items by specifying their coordinates:: - - >>> z.get_coordinate_selection(([1, 4], [1, 4])) - array([11, 44]) - - For convenience, the coordinate selection functionality is also available via the - `vindex` property, e.g.:: - - >>> z.vindex[[1, 4], [1, 4]] - array([11, 44]) - - Notes - ----- - Coordinate indexing is also known as point selection, and is a form of vectorized - or inner indexing. - - Slices are not supported. Coordinate arrays must be provided for all dimensions - of the array. - - Coordinate arrays may be multidimensional, in which case the output array will - also be multidimensional. Coordinate arrays are broadcast against each other - before being applied. The shape of the output will be the same as the shape of - each coordinate array after broadcasting. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, set_coordinate_selection, - get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # setup indexer - indexer = CoordinateIndexer(selection, self) - - # handle output - need to flatten - if out is not None: - out = out.reshape(-1) - - out = self._get_selection(indexer=indexer, out=out, fields=fields) - - # restore shape - out = out.reshape(indexer.sel_shape) - - return out - - def get_block_selection(self, selection, out=None, fields=None): - """Retrieve a selection of individual chunk blocks, by providing the indices - (coordinates) for each chunk block. - - Parameters - ---------- - selection : tuple - An integer (coordinate) or slice for each dimension of the array. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested selection. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.array(np.arange(100).reshape(10, 10), chunks=(3, 3)) - - Retrieve items by specifying their block coordinates:: - - >>> z.get_block_selection((1, slice(None))) - array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], - [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) - - Which is equivalent to:: - - >>> z[3:6, :] - array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], - [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) - - For convenience, the block selection functionality is also available via the - `blocks` property, e.g.:: - - >>> z.blocks[1] - array([[30, 31, 32, 33, 34, 35, 36, 37, 38, 39], - [40, 41, 42, 43, 44, 45, 46, 47, 48, 49], - [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]]) - - Notes - ----- - Block indexing is a convenience indexing method to work on individual chunks - with chunk index slicing. It has the same concept as Dask's `Array.blocks` - indexing. - - Slices are supported. However, only with a step size of one. - - Block index arrays may be multidimensional to index multidimensional arrays. - For example:: - - >>> z.blocks[0, 1:3] - array([[ 3, 4, 5, 6, 7, 8], - [13, 14, 15, 16, 17, 18], - [23, 24, 25, 26, 27, 28]]) - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - set_coordinate_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # setup indexer - indexer = BlockIndexer(selection, self) - - return self._get_selection(indexer=indexer, out=out, fields=fields) - - def get_mask_selection(self, selection, out=None, fields=None): - """Retrieve a selection of individual items, by providing a Boolean array of the - same shape as the array against which the selection is being made, where True - values indicate a selected item. - - Parameters - ---------- - selection : ndarray, bool - A Boolean array of the same shape as the array against which the selection is - being made. - out : ndarray, optional - If given, load the selected data directly into this array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to - extract data for. - - Returns - ------- - out : ndarray - A NumPy array containing the data for the requested selection. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.array(np.arange(100).reshape(10, 10)) - - Retrieve items by specifying a mask:: - - >>> sel = np.zeros_like(z, dtype=bool) - >>> sel[1, 1] = True - >>> sel[4, 4] = True - >>> z.get_mask_selection(sel) - array([11, 44]) - - For convenience, the mask selection functionality is also available via the - `vindex` property, e.g.:: - - >>> z.vindex[sel] - array([11, 44]) - - Notes - ----- - Mask indexing is a form of vectorized or inner indexing, and is equivalent to - coordinate indexing. Internally the mask array is converted to coordinate - arrays by calling `np.nonzero`. - - See Also - -------- - get_basic_selection, set_basic_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - set_coordinate_selection, get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - """ - - # refresh metadata - if not self._cache_metadata: - self._load_metadata() - - # check args - check_fields(fields, self._dtype) - - # setup indexer - indexer = MaskIndexer(selection, self) - - return self._get_selection(indexer=indexer, out=out, fields=fields) - - def _get_selection(self, indexer, out=None, fields=None): - # We iterate over all chunks which overlap the selection and thus contain data - # that needs to be extracted. Each chunk is processed in turn, extracting the - # necessary data and storing into the correct location in the output array. - - # N.B., it is an important optimisation that we only visit chunks which overlap - # the selection. This minimises the number of iterations in the main for loop. - - # check fields are sensible - out_dtype = check_fields(fields, self._dtype) - - # determine output shape - out_shape = indexer.shape - - # setup output array - if out is None: - out = np.empty_like( - self._meta_array, shape=out_shape, dtype=out_dtype, order=self._order - ) - else: - check_array_shape("out", out, out_shape) - - # iterate over chunks - - if math.prod(out_shape) > 0: - # allow storage to get multiple items at once - lchunk_coords, lchunk_selection, lout_selection = zip(*indexer) - self._chunk_getitems( - lchunk_coords, - lchunk_selection, - out, - lout_selection, - drop_axes=indexer.drop_axes, - fields=fields, - ) - if out.shape: - return out - else: - return out[()] - - def __setitem__(self, selection, value): - """Modify data for an item or region of the array. - - Parameters - ---------- - selection : tuple - An integer index or slice or tuple of int/slice specifying the requested - region for each dimension of the array. - value : scalar or array-like - Value to be stored into the array. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> z = zarr.v2.zeros(100, dtype=int) - - Set all array elements to the same scalar value:: - - >>> z[...] = 42 - >>> z[...] - array([42, 42, 42, ..., 42, 42, 42]) - - Set a portion of the array:: - - >>> z[:10] = np.arange(10) - >>> z[-10:] = np.arange(10)[::-1] - >>> z[...] - array([ 0, 1, 2, ..., 2, 1, 0]) - - Setup a 2-dimensional array:: - - >>> z = zarr.v2.zeros((5, 5), dtype=int) - - Set all array elements to the same scalar value:: - - >>> z[...] = 42 - - Set a portion of the array:: - - >>> z[0, :] = np.arange(z.shape[1]) - >>> z[:, 0] = np.arange(z.shape[0]) - >>> z[...] - array([[ 0, 1, 2, 3, 4], - [ 1, 42, 42, 42, 42], - [ 2, 42, 42, 42, 42], - [ 3, 42, 42, 42, 42], - [ 4, 42, 42, 42, 42]]) - - For arrays with a structured dtype, specific fields can be modified, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.v2.array(a) - >>> z['foo'] = b'zzz' - >>> z[...] - array([(b'zzz', 1, 4.2), (b'zzz', 2, 8.4), (b'zzz', 3, 12.6)], - dtype=[('foo', 'S3'), ('bar', ' 1 are supported, but slices with negative step are not. - - Currently the implementation for __setitem__ is provided by - :func:`vindex` if the indexing is pure fancy indexing (ie a - broadcast-compatible tuple of integer array indices), or by - :func:`set_basic_selection` otherwise. - - Effectively, this means that the following indexing modes are supported: - - - integer indexing - - slice indexing - - mixed slice and integer indexing - - boolean indexing - - fancy indexing (vectorized list of integers) - - For specific indexing options including outer indexing, see the - methods listed under See Also. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - set_orthogonal_selection, get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__ - - """ - fields, pure_selection = pop_fields(selection) - if is_pure_fancy_indexing(pure_selection, self.ndim): - self.vindex[selection] = value - elif is_pure_orthogonal_indexing(pure_selection, self.ndim): - self.set_orthogonal_selection(pure_selection, value, fields=fields) - else: - self.set_basic_selection(pure_selection, value, fields=fields) - - def set_basic_selection(self, selection, value, fields=None): - """Modify data for an item or region of the array. - - Parameters - ---------- - selection : tuple - An integer index or slice or tuple of int/slice specifying the requested - region for each dimension of the array. - value : scalar or array-like - Value to be stored into the array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to set - data for. - - Examples - -------- - Setup a 1-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.zeros(100, dtype=int) - - Set all array elements to the same scalar value:: - - >>> z.set_basic_selection(..., 42) - >>> z[...] - array([42, 42, 42, ..., 42, 42, 42]) - - Set a portion of the array:: - - >>> z.set_basic_selection(slice(10), np.arange(10)) - >>> z.set_basic_selection(slice(-10, None), np.arange(10)[::-1]) - >>> z[...] - array([ 0, 1, 2, ..., 2, 1, 0]) - - Setup a 2-dimensional array:: - - >>> z = zarr.v2.zeros((5, 5), dtype=int) - - Set all array elements to the same scalar value:: - - >>> z.set_basic_selection(..., 42) - - Set a portion of the array:: - - >>> z.set_basic_selection((0, slice(None)), np.arange(z.shape[1])) - >>> z.set_basic_selection((slice(None), 0), np.arange(z.shape[0])) - >>> z[...] - array([[ 0, 1, 2, 3, 4], - [ 1, 42, 42, 42, 42], - [ 2, 42, 42, 42, 42], - [ 3, 42, 42, 42, 42], - [ 4, 42, 42, 42, 42]]) - - For arrays with a structured dtype, the `fields` parameter can be used to set - data for a specific field, e.g.:: - - >>> a = np.array([(b'aaa', 1, 4.2), - ... (b'bbb', 2, 8.4), - ... (b'ccc', 3, 12.6)], - ... dtype=[('foo', 'S3'), ('bar', 'i4'), ('baz', 'f8')]) - >>> z = zarr.v2.array(a) - >>> z.set_basic_selection(slice(0, 2), b'zzz', fields='foo') - >>> z[:] - array([(b'zzz', 1, 4.2), (b'zzz', 2, 8.4), (b'ccc', 3, 12.6)], - dtype=[('foo', 'S3'), ('bar', '>> import zarr - >>> import numpy as np - >>> z = zarr.v2.zeros((5, 5), dtype=int) - - Set data for a selection of rows:: - - >>> z.set_orthogonal_selection(([1, 4], slice(None)), 1) - >>> z[...] - array([[0, 0, 0, 0, 0], - [1, 1, 1, 1, 1], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [1, 1, 1, 1, 1]]) - - Set data for a selection of columns:: - - >>> z.set_orthogonal_selection((slice(None), [1, 4]), 2) - >>> z[...] - array([[0, 2, 0, 0, 2], - [1, 2, 1, 1, 2], - [0, 2, 0, 0, 2], - [0, 2, 0, 0, 2], - [1, 2, 1, 1, 2]]) - - Set data for a selection of rows and columns:: - - >>> z.set_orthogonal_selection(([1, 4], [1, 4]), 3) - >>> z[...] - array([[0, 2, 0, 0, 2], - [1, 3, 1, 1, 3], - [0, 2, 0, 0, 2], - [0, 2, 0, 0, 2], - [1, 3, 1, 1, 3]]) - - For convenience, this functionality is also available via the `oindex` property. - E.g.:: - - >>> z.oindex[[1, 4], [1, 4]] = 4 - >>> z[...] - array([[0, 2, 0, 0, 2], - [1, 4, 1, 1, 4], - [0, 2, 0, 0, 2], - [0, 2, 0, 0, 2], - [1, 4, 1, 1, 4]]) - - Notes - ----- - Orthogonal indexing is also known as outer indexing. - - Slices with step > 1 are supported, but slices with negative step are not. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_coordinate_selection, set_coordinate_selection, get_orthogonal_selection, - get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - - # guard conditions - if self._read_only: - raise ReadOnlyError - - # refresh metadata - if not self._cache_metadata: - self._load_metadata_nosync() - - # setup indexer - indexer = OrthogonalIndexer(selection, self) - - self._set_selection(indexer, value, fields=fields) - - def set_coordinate_selection(self, selection, value, fields=None): - """Modify a selection of individual items, by providing the indices (coordinates) - for each item to be modified. - - Parameters - ---------- - selection : tuple - An integer (coordinate) array for each dimension of the array. - value : scalar or array-like - Value to be stored into the array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to set - data for. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.zeros((5, 5), dtype=int) - - Set data for a selection of items:: - - >>> z.set_coordinate_selection(([1, 4], [1, 4]), 1) - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 1]]) - - For convenience, this functionality is also available via the `vindex` property. - E.g.:: - - >>> z.vindex[[1, 4], [1, 4]] = 2 - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 2, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 2]]) - - Notes - ----- - Coordinate indexing is also known as point selection, and is a form of vectorized - or inner indexing. - - Slices are not supported. Coordinate arrays must be provided for all dimensions - of the array. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - - # guard conditions - if self._read_only: - raise ReadOnlyError - - # refresh metadata - if not self._cache_metadata: - self._load_metadata_nosync() - - # setup indexer - indexer = CoordinateIndexer(selection, self) - - # handle value - need ndarray-like flatten value - if not is_scalar(value, self._dtype): - try: - value = ensure_ndarray_like(value) - except TypeError: - # Handle types like `list` or `tuple` - value = np.array(value, like=self._meta_array) - if hasattr(value, "shape") and len(value.shape) > 1: - value = value.reshape(-1) - - self._set_selection(indexer, value, fields=fields) - - def set_block_selection(self, selection, value, fields=None): - """Modify a selection of individual blocks, by providing the chunk indices - (coordinates) for each block to be modified. - - Parameters - ---------- - selection : tuple - An integer (coordinate) or slice for each dimension of the array. - value : scalar or array-like - Value to be stored into the array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to set - data for. - - Examples - -------- - Set up a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.zeros((6, 6), dtype=int, chunks=2) - - Set data for a selection of items:: - - >>> z.set_block_selection((1, 0), 1) - >>> z[...] - array([[0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0]]) - - For convenience, this functionality is also available via the `blocks` property. - E.g.:: - - >>> z.blocks[2, 1] = 4 - >>> z[...] - array([[0, 0, 0, 0, 0, 0], - [0, 0, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [1, 1, 0, 0, 0, 0], - [0, 0, 4, 4, 0, 0], - [0, 0, 4, 4, 0, 0]]) - - >>> z.blocks[:, 2] = 7 - >>> z[...] - array([[0, 0, 0, 0, 7, 7], - [0, 0, 0, 0, 7, 7], - [1, 1, 0, 0, 7, 7], - [1, 1, 0, 0, 7, 7], - [0, 0, 4, 4, 7, 7], - [0, 0, 4, 4, 7, 7]]) - - Notes - ----- - Block indexing is a convenience indexing method to work on individual chunks - with chunk index slicing. It has the same concept as Dask's `Array.blocks` - indexing. - - Slices are supported. However, only with a step size of one. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, set_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - # guard conditions - if self._read_only: - raise ReadOnlyError - - # refresh metadata - if not self._cache_metadata: - self._load_metadata_nosync() - - # setup indexer - indexer = BlockIndexer(selection, self) - - self._set_selection(indexer, value, fields=fields) - - def set_mask_selection(self, selection, value, fields=None): - """Modify a selection of individual items, by providing a Boolean array of the - same shape as the array against which the selection is being made, where True - values indicate a selected item. - - Parameters - ---------- - selection : ndarray, bool - A Boolean array of the same shape as the array against which the selection is - being made. - value : scalar or array-like - Value to be stored into the array. - fields : str or sequence of str, optional - For arrays with a structured dtype, one or more fields can be specified to set - data for. - - Examples - -------- - Setup a 2-dimensional array:: - - >>> import zarr - >>> import numpy as np - >>> z = zarr.v2.zeros((5, 5), dtype=int) - - Set data for a selection of items:: - - >>> sel = np.zeros_like(z, dtype=bool) - >>> sel[1, 1] = True - >>> sel[4, 4] = True - >>> z.set_mask_selection(sel, 1) - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 1, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 1]]) - - For convenience, this functionality is also available via the `vindex` property. - E.g.:: - - >>> z.vindex[sel] = 2 - >>> z[...] - array([[0, 0, 0, 0, 0], - [0, 2, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 0], - [0, 0, 0, 0, 2]]) - - Notes - ----- - Mask indexing is a form of vectorized or inner indexing, and is equivalent to - coordinate indexing. Internally the mask array is converted to coordinate - arrays by calling `np.nonzero`. - - See Also - -------- - get_basic_selection, set_basic_selection, get_mask_selection, - get_orthogonal_selection, set_orthogonal_selection, get_coordinate_selection, - set_coordinate_selection, get_block_selection, set_block_selection, - vindex, oindex, blocks, __getitem__, __setitem__ - - """ - - # guard conditions - if self._read_only: - raise ReadOnlyError - - # refresh metadata - if not self._cache_metadata: - self._load_metadata_nosync() - - # setup indexer - indexer = MaskIndexer(selection, self) - - self._set_selection(indexer, value, fields=fields) - - def _set_basic_selection_zd(self, selection, value, fields=None): - # special case __setitem__ for zero-dimensional array - - # check selection is valid - selection = ensure_tuple(selection) - if selection not in ((), (Ellipsis,)): - err_too_many_indices(selection, self._shape) - - # check fields - check_fields(fields, self._dtype) - fields = check_no_multi_fields(fields) - - # obtain key for chunk - ckey = self._chunk_key((0,)) - - # setup chunk - try: - # obtain compressed data for chunk - cdata = self.chunk_store[ckey] - - except KeyError: - # chunk not initialized - chunk = np.zeros_like(self._meta_array, shape=(), dtype=self._dtype) - if self._fill_value is not None: - chunk.fill(self._fill_value) - - else: - # decode chunk - chunk = self._decode_chunk(cdata).copy() - - # set value - if fields: - chunk[fields][selection] = value - else: - chunk[selection] = value - - # remove chunk if write_empty_chunks is false and it only contains the fill value - if (not self.write_empty_chunks) and all_equal(self.fill_value, chunk): - try: - del self.chunk_store[ckey] - return - except Exception: # pragma: no cover - # deleting failed, fallback to overwriting - pass - else: - # encode and store - cdata = self._encode_chunk(chunk) - self.chunk_store[ckey] = cdata - - def _set_basic_selection_nd(self, selection, value, fields=None): - # implementation of __setitem__ for array with at least one dimension - - # setup indexer - indexer = BasicIndexer(selection, self) - - self._set_selection(indexer, value, fields=fields) - - def _set_selection(self, indexer, value, fields=None): - # We iterate over all chunks which overlap the selection and thus contain data - # that needs to be replaced. Each chunk is processed in turn, extracting the - # necessary data from the value array and storing into the chunk array. - - # N.B., it is an important optimisation that we only visit chunks which overlap - # the selection. This minimises the number of iterations in the main for loop. - - # check fields are sensible - check_fields(fields, self._dtype) - fields = check_no_multi_fields(fields) - - # determine indices of chunks overlapping the selection - sel_shape = indexer.shape - - # check value shape - if sel_shape == (): - # setting a single item - pass - elif is_scalar(value, self._dtype): - # setting a scalar value - pass - else: - if not hasattr(value, "shape"): - value = np.asanyarray(value, like=self._meta_array) - check_array_shape("value", value, sel_shape) - - # iterate over chunks in range - if ( - not hasattr(self.chunk_store, "setitems") - or self._synchronizer is not None - or any(map(lambda x: x == 0, self.shape)) - ): - # iterative approach - for chunk_coords, chunk_selection, out_selection in indexer: - # extract data to store - if sel_shape == (): - chunk_value = value - elif is_scalar(value, self._dtype): - chunk_value = value - else: - chunk_value = value[out_selection] - # handle missing singleton dimensions - if indexer.drop_axes: - item = [slice(None)] * self.ndim - for a in indexer.drop_axes: - item[a] = np.newaxis - item = tuple(item) - chunk_value = chunk_value[item] - - # put data - self._chunk_setitem(chunk_coords, chunk_selection, chunk_value, fields=fields) - else: - lchunk_coords, lchunk_selection, lout_selection = zip(*indexer) - chunk_values = [] - for out_selection in lout_selection: - if sel_shape == (): - chunk_values.append(value) - elif is_scalar(value, self._dtype): - chunk_values.append(value) - else: - cv = value[out_selection] - # handle missing singleton dimensions - if indexer.drop_axes: # pragma: no cover - item = [slice(None)] * self.ndim - for a in indexer.drop_axes: - item[a] = np.newaxis - item = tuple(item) - cv = chunk_value[item] - chunk_values.append(cv) - - self._chunk_setitems(lchunk_coords, lchunk_selection, chunk_values, fields=fields) - - def _process_chunk( - self, - out, - cdata, - chunk_selection, - drop_axes, - out_is_ndarray, - fields, - out_selection, - partial_read_decode=False, - ): - """Take binary data from storage and fill output array""" - if ( - out_is_ndarray - and not fields - and is_contiguous_selection(out_selection) - and is_total_slice(chunk_selection, self._chunks) - and not self._filters - and self._dtype != object - ): - dest = out[out_selection] - # Assume that array-like objects that doesn't have a - # `writeable` flag is writable. - dest_is_writable = getattr(dest, "writeable", True) - write_direct = dest_is_writable and ( - (self._order == "C" and dest.flags.c_contiguous) - or (self._order == "F" and dest.flags.f_contiguous) - ) - - if write_direct: - # optimization: we want the whole chunk, and the destination is - # contiguous, so we can decompress directly from the chunk - # into the destination array - if self._compressor: - if isinstance(cdata, PartialReadBuffer): - cdata = cdata.read_full() - self._compressor.decode(cdata, dest) - else: - chunk = ensure_ndarray_like(cdata).view(self._dtype) - chunk = chunk.reshape(self._chunks, order=self._order) - np.copyto(dest, chunk) - return - - # decode chunk - try: - if partial_read_decode: - cdata.prepare_chunk() - # size of chunk - tmp = np.empty_like(self._meta_array, shape=self._chunks, dtype=self.dtype) - index_selection = PartialChunkIterator(chunk_selection, self.chunks) - for start, nitems, partial_out_selection in index_selection: - expected_shape = [ - len(range(*partial_out_selection[i].indices(self.chunks[0] + 1))) - if i < len(partial_out_selection) - else dim - for i, dim in enumerate(self.chunks) - ] - cdata.read_part(start, nitems) - chunk_partial = self._decode_chunk( - cdata.buff, - start=start, - nitems=nitems, - expected_shape=expected_shape, - ) - tmp[partial_out_selection] = chunk_partial - out[out_selection] = tmp[chunk_selection] - return - except ArrayIndexError: - cdata = cdata.read_full() - chunk = self._decode_chunk(cdata) - - # select data from chunk - if fields: - chunk = chunk[fields] - tmp = chunk[chunk_selection] - if drop_axes: - tmp = np.squeeze(tmp, axis=drop_axes) - - # store selected data in output - out[out_selection] = tmp - - def _chunk_getitems( - self, lchunk_coords, lchunk_selection, out, lout_selection, drop_axes=None, fields=None - ): - """Obtain part or whole of chunks. - - Parameters - ---------- - chunk_coords : list of tuple of ints - Indices of the chunks. - chunk_selection : list of selections - Location of region within the chunks to extract. - out : ndarray - Array to store result in. - out_selection : list of selections - Location of regions within output array to store results in. - drop_axes : tuple of ints - Axes to squeeze out of the chunk. - fields - TODO - """ - - out_is_ndarray = True - try: - out = ensure_ndarray_like(out) - except TypeError: # pragma: no cover - out_is_ndarray = False - - # Keys to retrieve - ckeys = [self._chunk_key(ch) for ch in lchunk_coords] - - # Check if we can do a partial read - if ( - self._partial_decompress - and self._compressor - and self._compressor.codec_id == "blosc" - and hasattr(self._compressor, "decode_partial") - and not fields - and self.dtype != object - and hasattr(self.chunk_store, "getitems") - ): - partial_read_decode = True - cdatas = { - ckey: PartialReadBuffer(ckey, self.chunk_store) - for ckey in ckeys - if ckey in self.chunk_store - } - elif ( - self._partial_decompress - and not self._compressor - and not fields - and self.dtype != object - and hasattr(self.chunk_store, "get_partial_values") - and self.chunk_store.supports_efficient_get_partial_values - ): - partial_read_decode = True - cdatas = { - ckey: UncompressedPartialReadBufferV3( - ckey, self.chunk_store, itemsize=self.itemsize - ) - for ckey in ckeys - if ckey in self.chunk_store - } - elif hasattr(self.chunk_store, "get_partial_values"): - partial_read_decode = False - values = self.chunk_store.get_partial_values([(ckey, (0, None)) for ckey in ckeys]) - cdatas = {key: value for key, value in zip(ckeys, values) if value is not None} - else: - partial_read_decode = False - contexts = {} - if not isinstance(self._meta_array, np.ndarray): - contexts = ConstantMap(ckeys, constant=Context(meta_array=self._meta_array)) - cdatas = self.chunk_store.getitems(ckeys, contexts=contexts) - - for ckey, chunk_select, out_select in zip(ckeys, lchunk_selection, lout_selection): - if ckey in cdatas: - self._process_chunk( - out, - cdatas[ckey], - chunk_select, - drop_axes, - out_is_ndarray, - fields, - out_select, - partial_read_decode=partial_read_decode, - ) - else: - # check exception type - if self._fill_value is not None: - if fields: - fill_value = self._fill_value[fields] - else: - fill_value = self._fill_value - out[out_select] = fill_value - - def _chunk_setitems(self, lchunk_coords, lchunk_selection, values, fields=None): - ckeys = map(self._chunk_key, lchunk_coords) - cdatas = { - key: self._process_for_setitem(key, sel, val, fields=fields) - for key, sel, val in zip(ckeys, lchunk_selection, values) - } - to_store = {} - if not self.write_empty_chunks: - empty_chunks = {k: v for k, v in cdatas.items() if all_equal(self.fill_value, v)} - self._chunk_delitems(empty_chunks.keys()) - nonempty_keys = cdatas.keys() - empty_chunks.keys() - to_store = {k: self._encode_chunk(cdatas[k]) for k in nonempty_keys} - else: - to_store = {k: self._encode_chunk(v) for k, v in cdatas.items()} - self.chunk_store.setitems(to_store) - - def _chunk_delitems(self, ckeys): - if hasattr(self.store, "delitems"): - self.store.delitems(ckeys) - else: # pragma: no cover - # exempting this branch from coverage as there are no extant stores - # that will trigger this condition, but it's possible that they - # will be developed in the future. - tuple(map(self._chunk_delitem, ckeys)) - - def _chunk_delitem(self, ckey): - """ - Attempt to delete the value associated with ckey. - """ - try: - del self.chunk_store[ckey] - except KeyError: - pass - - def _chunk_setitem(self, chunk_coords, chunk_selection, value, fields=None): - """Replace part or whole of a chunk. - - Parameters - ---------- - chunk_coords : tuple of ints - Indices of the chunk. - chunk_selection : tuple of slices - Location of region within the chunk. - value : scalar or ndarray - Value to set. - - """ - - if self._synchronizer is None: - # no synchronization - lock = nolock - else: - # synchronize on the chunk - ckey = self._chunk_key(chunk_coords) - lock = self._synchronizer[ckey] - - with lock: - self._chunk_setitem_nosync(chunk_coords, chunk_selection, value, fields=fields) - - def _chunk_setitem_nosync(self, chunk_coords, chunk_selection, value, fields=None): - ckey = self._chunk_key(chunk_coords) - cdata = self._process_for_setitem(ckey, chunk_selection, value, fields=fields) - - # attempt to delete chunk if it only contains the fill value - if (not self.write_empty_chunks) and all_equal(self.fill_value, cdata): - self._chunk_delitem(ckey) - else: - self.chunk_store[ckey] = self._encode_chunk(cdata) - - def _process_for_setitem(self, ckey, chunk_selection, value, fields=None): - if is_total_slice(chunk_selection, self._chunks) and not fields: - # totally replace chunk - - # optimization: we are completely replacing the chunk, so no need - # to access the existing chunk data - - if is_scalar(value, self._dtype): - # setup array filled with value - chunk = np.empty_like( - self._meta_array, shape=self._chunks, dtype=self._dtype, order=self._order - ) - chunk.fill(value) - - else: - # ensure array is contiguous - chunk = value.astype(self._dtype, order=self._order, copy=False) - - else: - # partially replace the contents of this chunk - - try: - # obtain compressed data for chunk - cdata = self.chunk_store[ckey] - - except KeyError: - # chunk not initialized - if self._fill_value is not None: - chunk = np.empty_like( - self._meta_array, shape=self._chunks, dtype=self._dtype, order=self._order - ) - chunk.fill(self._fill_value) - elif self._dtype == object: - chunk = np.empty(self._chunks, dtype=self._dtype, order=self._order) - else: - # N.B., use zeros here so any region beyond the array has consistent - # and compressible data - chunk = np.zeros_like( - self._meta_array, shape=self._chunks, dtype=self._dtype, order=self._order - ) - - else: - # decode chunk - chunk = self._decode_chunk(cdata) - if not chunk.flags.writeable: - chunk = chunk.copy(order="K") - - # modify - if fields: - # N.B., currently multi-field assignment is not supported in numpy, so - # this only works for a single field - chunk[fields][chunk_selection] = value - else: - chunk[chunk_selection] = value - - return chunk - - def _chunk_key(self, chunk_coords): - return self._key_prefix + self._dimension_separator.join(map(str, chunk_coords)) - - def _decode_chunk(self, cdata, start=None, nitems=None, expected_shape=None): - # decompress - if self._compressor: - # only decode requested items - if ( - all(x is not None for x in [start, nitems]) and self._compressor.codec_id == "blosc" - ) and hasattr(self._compressor, "decode_partial"): - chunk = self._compressor.decode_partial(cdata, start, nitems) - else: - chunk = self._compressor.decode(cdata) - else: - chunk = cdata - - # apply filters - if self._filters: - for f in reversed(self._filters): - chunk = f.decode(chunk) - - # view as numpy array with correct dtype - chunk = ensure_ndarray_like(chunk) - # special case object dtype, because incorrect handling can lead to - # segfaults and other bad things happening - if self._dtype != object: - chunk = chunk.view(self._dtype) - elif chunk.dtype != object: - # If we end up here, someone must have hacked around with the filters. - # We cannot deal with object arrays unless there is an object - # codec in the filter chain, i.e., a filter that converts from object - # array to something else during encoding, and converts back to object - # array during decoding. - raise RuntimeError("cannot read object array without object codec") - - # ensure correct chunk shape - chunk = chunk.reshape(-1, order="A") - chunk = chunk.reshape(expected_shape or self._chunks, order=self._order) - - return chunk - - def _encode_chunk(self, chunk): - # apply filters - if self._filters: - for f in self._filters: - chunk = f.encode(chunk) - - # check object encoding - if ensure_ndarray_like(chunk).dtype == object: - raise RuntimeError("cannot write object array without object codec") - - # compress - if self._compressor: - cdata = self._compressor.encode(chunk) - else: - cdata = chunk - - # ensure in-memory data is immutable and easy to compare - if isinstance(self.chunk_store, KVStore) or isinstance(self._chunk_store, KVStore): - cdata = ensure_bytes(cdata) - - return cdata - - def __repr__(self): - t = type(self) - r = "<{}.{}".format(t.__module__, t.__name__) - if self.name: - r += " %r" % self.name - r += " %s" % str(self.shape) - r += " %s" % self.dtype - if self._read_only: - r += " read-only" - r += ">" - return r - - @property - def info(self): - """Report some diagnostic information about the array. - - Examples - -------- - >>> import zarr - >>> z = zarr.v2.zeros(1000000, chunks=100000, dtype='i4') - >>> z.info - Type : zarr.v2.core.Array - Data type : int32 - Shape : (1000000,) - Chunk shape : (100000,) - Order : C - Read-only : False - Compressor : Blosc(cname='lz4', clevel=5, shuffle=SHUFFLE, blocksize=0) - Store type : zarr.v2.storage.KVStore - No. bytes : 4000000 (3.8M) - No. bytes stored : 320 - Storage ratio : 12500.0 - Chunks initialized : 0/10 - - """ - return self._info_reporter - - def info_items(self): - return self._synchronized_op(self._info_items_nosync) - - def _info_items_nosync(self): - def typestr(o): - return "{}.{}".format(type(o).__module__, type(o).__name__) - - def bytestr(n): - if n > 2**10: - return "{} ({})".format(n, human_readable_size(n)) - else: - return str(n) - - items = [] - - # basic info - if self.name is not None: - items += [("Name", self.name)] - items += [ - ("Type", typestr(self)), - ("Data type", "%s" % self.dtype), - ("Shape", str(self.shape)), - ("Chunk shape", str(self.chunks)), - ("Order", self.order), - ("Read-only", str(self.read_only)), - ] - - # filters - if self.filters: - for i, f in enumerate(self.filters): - items += [("Filter [%s]" % i, repr(f))] - - # compressor - items += [("Compressor", repr(self.compressor))] - - # synchronizer - if self._synchronizer is not None: - items += [("Synchronizer type", typestr(self._synchronizer))] - - # storage info - items += [("Store type", typestr(self._store))] - if self._chunk_store is not None: - items += [("Chunk store type", typestr(self._chunk_store))] - items += [("No. bytes", bytestr(self.nbytes))] - if self.nbytes_stored > 0: - items += [ - ("No. bytes stored", bytestr(self.nbytes_stored)), - ("Storage ratio", "%.1f" % (self.nbytes / self.nbytes_stored)), - ] - items += [("Chunks initialized", "{}/{}".format(self.nchunks_initialized, self.nchunks))] - - return items - - def digest(self, hashname="sha1"): - """ - Compute a checksum for the data. Default uses sha1 for speed. - - Examples - -------- - >>> import binascii - >>> import zarr - >>> z = zarr.v2.empty(shape=(10000, 10000), chunks=(1000, 1000)) - >>> binascii.hexlify(z.digest()) - b'041f90bc7a571452af4f850a8ca2c6cddfa8a1ac' - >>> z = zarr.v2.zeros(shape=(10000, 10000), chunks=(1000, 1000)) - >>> binascii.hexlify(z.digest()) - b'7162d416d26a68063b66ed1f30e0a866e4abed60' - >>> z = zarr.v2.zeros(shape=(10000, 10000), dtype="u1", chunks=(1000, 1000)) - >>> binascii.hexlify(z.digest()) - b'cb387af37410ae5a3222e893cf3373e4e4f22816' - """ - - h = hashlib.new(hashname) - - for i in itertools.product(*[range(s) for s in self.cdata_shape]): - h.update(self.chunk_store.get(self._chunk_key(i), b"")) - - mkey = _prefix_to_array_key(self._store, self._key_prefix) - h.update(self.store.get(mkey, b"")) - - h.update(self.store.get(self.attrs.key, b"")) - - checksum = h.digest() - - return checksum - - def hexdigest(self, hashname="sha1"): - """ - Compute a checksum for the data. Default uses sha1 for speed. - - Examples - -------- - >>> import zarr - >>> z = zarr.v2.empty(shape=(10000, 10000), chunks=(1000, 1000)) - >>> z.hexdigest() - '041f90bc7a571452af4f850a8ca2c6cddfa8a1ac' - >>> z = zarr.v2.zeros(shape=(10000, 10000), chunks=(1000, 1000)) - >>> z.hexdigest() - '7162d416d26a68063b66ed1f30e0a866e4abed60' - >>> z = zarr.v2.zeros(shape=(10000, 10000), dtype="u1", chunks=(1000, 1000)) - >>> z.hexdigest() - 'cb387af37410ae5a3222e893cf3373e4e4f22816' - """ - - checksum = binascii.hexlify(self.digest(hashname=hashname)) - - # This is a bytes object on Python 3 and we want a str. - if not isinstance(checksum, str): - checksum = checksum.decode("utf8") - - return checksum - - def __getstate__(self): - return { - "store": self._store, - "path": self._path, - "read_only": self._read_only, - "chunk_store": self._chunk_store, - "synchronizer": self._synchronizer, - "cache_metadata": self._cache_metadata, - "cache_attrs": self._attrs.cache, - "partial_decompress": self._partial_decompress, - "write_empty_chunks": self._write_empty_chunks, - "meta_array": self._meta_array, - } - - def __setstate__(self, state): - self.__init__(**state) - - def _synchronized_op(self, f, *args, **kwargs): - if self._synchronizer is None: - # no synchronization - lock = nolock - - else: - # synchronize on the array - mkey = _prefix_to_array_key(self._store, self._key_prefix) - lock = self._synchronizer[mkey] - - with lock: - self._refresh_metadata_nosync() - result = f(*args, **kwargs) - - return result - - def _write_op(self, f, *args, **kwargs): - # guard condition - if self._read_only: - raise ReadOnlyError - - return self._synchronized_op(f, *args, **kwargs) - - def resize(self, *args): - """Change the shape of the array by growing or shrinking one or more - dimensions. - - Examples - -------- - >>> import zarr - >>> z = zarr.v2.zeros(shape=(10000, 10000), chunks=(1000, 1000)) - >>> z.shape - (10000, 10000) - >>> z.resize(20000, 10000) - >>> z.shape - (20000, 10000) - >>> z.resize(30000, 1000) - >>> z.shape - (30000, 1000) - - Notes - ----- - When resizing an array, the data are not rearranged in any way. - - If one or more dimensions are shrunk, any chunks falling outside the - new array shape will be deleted from the underlying store. - However, it is noteworthy that the chunks partially falling inside the new array - (i.e. boundary chunks) will remain intact, and therefore, - the data falling outside the new array but inside the boundary chunks - would be restored by a subsequent resize operation that grows the array size. - - """ - - return self._write_op(self._resize_nosync, *args) - - def _resize_nosync(self, *args): - # normalize new shape argument - old_shape = self._shape - new_shape = normalize_resize_args(old_shape, *args) - old_cdata_shape = self._cdata_shape - - # update metadata - self._shape = new_shape - self._flush_metadata_nosync() - - # determine the new number and arrangement of chunks - chunks = self._chunks - new_cdata_shape = tuple(math.ceil(s / c) for s, c in zip(new_shape, chunks)) - - # remove any chunks not within range - # The idea is that, along each dimension, - # only find and remove the chunk slices that exist in 'old' but not 'new' data. - # Note that a mutable list ('old_cdata_shape_working_list') is introduced here - # to dynamically adjust the number of chunks along the already-processed dimensions - # in order to avoid duplicate chunk removal. - chunk_store = self.chunk_store - old_cdata_shape_working_list = list(old_cdata_shape) - for idx_cdata, (val_old_cdata, val_new_cdata) in enumerate( - zip(old_cdata_shape_working_list, new_cdata_shape) - ): - for cidx in itertools.product( - *[ - range(n_new, n_old) if (idx == idx_cdata) else range(n_old) - for idx, (n_old, n_new) in enumerate( - zip(old_cdata_shape_working_list, new_cdata_shape) - ) - ] - ): - key = self._chunk_key(cidx) - try: - del chunk_store[key] - except KeyError: - # chunk not initialized - pass - old_cdata_shape_working_list[idx_cdata] = min(val_old_cdata, val_new_cdata) - - def append(self, data, axis=0): - """Append `data` to `axis`. - - Parameters - ---------- - data : array-like - Data to be appended. - axis : int - Axis along which to append. - - Returns - ------- - new_shape : tuple - - Notes - ----- - The size of all dimensions other than `axis` must match between this - array and `data`. - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> a = np.arange(10000000, dtype='i4').reshape(10000, 1000) - >>> z = zarr.v2.array(a, chunks=(1000, 100)) - >>> z.shape - (10000, 1000) - >>> z.append(a) - (20000, 1000) - >>> z.append(np.vstack([a, a]), axis=1) - (20000, 2000) - >>> z.shape - (20000, 2000) - - """ - return self._write_op(self._append_nosync, data, axis=axis) - - def _append_nosync(self, data, axis=0): - # ensure data is array-like - if not hasattr(data, "shape"): - data = np.asanyarray(data, like=self._meta_array) - - # ensure shapes are compatible for non-append dimensions - self_shape_preserved = tuple(s for i, s in enumerate(self._shape) if i != axis) - data_shape_preserved = tuple(s for i, s in enumerate(data.shape) if i != axis) - if self_shape_preserved != data_shape_preserved: - raise ValueError( - "shape of data to append is not compatible with the array; " - "all dimensions must match except for the dimension being " - "appended" - ) - - # remember old shape - old_shape = self._shape - - # determine new shape - new_shape = tuple( - self._shape[i] if i != axis else self._shape[i] + data.shape[i] - for i in range(len(self._shape)) - ) - - # resize - self._resize_nosync(new_shape) - - # store data - # noinspection PyTypeChecker - append_selection = tuple( - slice(None) if i != axis else slice(old_shape[i], new_shape[i]) - for i in range(len(self._shape)) - ) - self[append_selection] = data - - return new_shape - - def view( - self, - shape=None, - chunks=None, - dtype=None, - fill_value=None, - filters=None, - read_only=None, - synchronizer=None, - ): - """Return an array sharing the same data. - - Parameters - ---------- - shape : int or tuple of ints - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. - dtype : string or dtype, optional - NumPy dtype. - fill_value : object - Default value to use for uninitialized portions of the array. - filters : sequence, optional - Sequence of filters to use to encode chunk data prior to - compression. - read_only : bool, optional - True if array should be protected against modification. - synchronizer : object, optional - Array synchronizer. - - Notes - ----- - WARNING: This is an experimental feature and should be used with care. - There are plenty of ways to generate errors and/or cause data - corruption. - - Examples - -------- - - Bypass filters: - - >>> import zarr - >>> import numpy as np - >>> np.random.seed(42) - >>> labels = ['female', 'male'] - >>> data = np.random.choice(labels, size=10000) - >>> filters = [zarr.v2.Categorize(labels=labels, - ... dtype=data.dtype, - ... astype='u1')] - >>> a = zarr.v2.array(data, chunks=1000, filters=filters) - >>> a[:] - array(['female', 'male', 'female', ..., 'male', 'male', 'female'], - dtype='>> v = a.view(dtype='u1', filters=[]) - >>> v.is_view - True - >>> v[:] - array([1, 2, 1, ..., 2, 2, 1], dtype=uint8) - - Views can be used to modify data: - - >>> x = v[:] - >>> x.sort() - >>> v[:] = x - >>> v[:] - array([1, 1, 1, ..., 2, 2, 2], dtype=uint8) - >>> a[:] - array(['female', 'female', 'female', ..., 'male', 'male', 'male'], - dtype='>> data = np.random.randint(0, 2, size=10000, dtype='u1') - >>> a = zarr.v2.array(data, chunks=1000) - >>> a[:] - array([0, 0, 1, ..., 1, 0, 0], dtype=uint8) - >>> v = a.view(dtype=bool) - >>> v[:] - array([False, False, True, ..., True, False, False]) - >>> np.all(a[:].view(dtype=bool) == v[:]) - True - - An array can be viewed with a dtype with a different item size, however - some care is needed to adjust the shape and chunk shape so that chunk - data is interpreted correctly: - - >>> data = np.arange(10000, dtype='u2') - >>> a = zarr.v2.array(data, chunks=1000) - >>> a[:10] - array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=uint16) - >>> v = a.view(dtype='u1', shape=20000, chunks=2000) - >>> v[:10] - array([0, 0, 1, 0, 2, 0, 3, 0, 4, 0], dtype=uint8) - >>> np.all(a[:].view('u1') == v[:]) - True - - Change fill value for uninitialized chunks: - - >>> a = zarr.v2.full(10000, chunks=1000, fill_value=-1, dtype='i1') - >>> a[:] - array([-1, -1, -1, ..., -1, -1, -1], dtype=int8) - >>> v = a.view(fill_value=42) - >>> v[:] - array([42, 42, 42, ..., 42, 42, 42], dtype=int8) - - Note that resizing or appending to views is not permitted: - - >>> a = zarr.v2.empty(10000) - >>> v = a.view() - >>> try: - ... v.resize(20000) - ... except PermissionError as e: - ... print(e) - operation not permitted for views - - """ - - store = self._store - chunk_store = self._chunk_store - path = self._path - if read_only is None: - read_only = self._read_only - if synchronizer is None: - synchronizer = self._synchronizer - a = Array( - store=store, - path=path, - chunk_store=chunk_store, - read_only=read_only, - synchronizer=synchronizer, - cache_metadata=True, - ) - a._is_view = True - - # allow override of some properties - if dtype is None: - dtype = self._dtype - else: - dtype = np.dtype(dtype) - a._dtype = dtype - if shape is None: - shape = self._shape - else: - shape = normalize_shape(shape) - a._shape = shape - if chunks is not None: - chunks = normalize_chunks(chunks, shape, dtype.itemsize) - a._chunks = chunks - if fill_value is not None: - a._fill_value = fill_value - if filters is not None: - a._filters = filters - - return a - - def astype(self, dtype): - """Returns a view that does on the fly type conversion of the underlying data. - - Parameters - ---------- - dtype : string or dtype - NumPy dtype. - - Notes - ----- - This method returns a new Array object which is a view on the same - underlying chunk data. Modifying any data via the view is currently - not permitted and will result in an error. This is an experimental - feature and its behavior is subject to change in the future. - - See Also - -------- - Array.view - - Examples - -------- - - >>> import zarr - >>> import numpy as np - >>> data = np.arange(100, dtype=np.uint8) - >>> a = zarr.v2.array(data, chunks=10) - >>> a[:] - array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, - 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, - 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, - 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - 96, 97, 98, 99], dtype=uint8) - >>> v = a.astype(np.float32) - >>> v.is_view - True - >>> v[:] - array([ 0., 1., 2., 3., 4., 5., 6., 7., 8., 9., - 10., 11., 12., 13., 14., 15., 16., 17., 18., 19., - 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., - 30., 31., 32., 33., 34., 35., 36., 37., 38., 39., - 40., 41., 42., 43., 44., 45., 46., 47., 48., 49., - 50., 51., 52., 53., 54., 55., 56., 57., 58., 59., - 60., 61., 62., 63., 64., 65., 66., 67., 68., 69., - 70., 71., 72., 73., 74., 75., 76., 77., 78., 79., - 80., 81., 82., 83., 84., 85., 86., 87., 88., 89., - 90., 91., 92., 93., 94., 95., 96., 97., 98., 99.], - dtype=float32) - """ - - dtype = np.dtype(dtype) - - filters = [] - if self._filters: - filters.extend(self._filters) - filters.insert(0, AsType(encode_dtype=self._dtype, decode_dtype=dtype)) - - return self.view(filters=filters, dtype=dtype, read_only=True) diff --git a/src/zarr/v2/creation.py b/src/zarr/v2/creation.py deleted file mode 100644 index d0ba00603d..0000000000 --- a/src/zarr/v2/creation.py +++ /dev/null @@ -1,707 +0,0 @@ -from typing import Optional -from warnings import warn - -import numpy as np -from numcodecs.registry import codec_registry - -from zarr.v2.core import Array -from zarr.v2.errors import ( - ArrayNotFoundError, - ContainsArrayError, - ContainsGroupError, -) -from zarr.v2.storage import ( - contains_array, - contains_group, - default_compressor, - init_array, - normalize_storage_path, - normalize_store_arg, -) -from zarr.v2.util import normalize_dimension_separator - - -def create( - shape, - chunks=True, - dtype=None, - compressor="default", - fill_value: Optional[int] = 0, - order="C", - store=None, - synchronizer=None, - overwrite=False, - path=None, - chunk_store=None, - filters=None, - cache_metadata=True, - cache_attrs=True, - read_only=False, - object_codec=None, - dimension_separator=None, - write_empty_chunks=True, - *, - meta_array=None, - **kwargs, -): - """Create an array. - - Parameters - ---------- - shape : int or tuple of ints - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. If True, will be guessed from `shape` and `dtype`. If - False, will be set to `shape`, i.e., single chunk for the whole array. - If an int, the chunk size in each dimension will be given by the value - of `chunks`. Default is True. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - store : MutableMapping or string - Store or path to directory in file system or name of zip file. - synchronizer : object, optional - Array synchronizer. - overwrite : bool, optional - If True, delete all pre-existing data in `store` at `path` before - creating the array. - path : string, optional - Path under which array is stored. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - filters : sequence of Codecs, optional - Sequence of filters to use to encode chunk data prior to compression. - cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - read_only : bool, optional - True if array should be protected against modification. - object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - .. versionadded:: 2.8 - - write_empty_chunks : bool, optional - If True (default), all chunks will be stored regardless of their - contents. If False, each chunk is compared to the array's fill value - prior to storing. If a chunk is uniformly equal to the fill value, then - that chunk is not be stored, and the store entry for that chunk's key - is deleted. This setting enables sparser storage, as only chunks with - non-fill-value data are stored, at the expense of overhead associated - with checking the data of each chunk. - - .. versionadded:: 2.11 - - meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. - - .. versionadded:: 2.13 - - Returns - ------- - z : zarr.v2.core.Array - - Examples - -------- - - Create an array with default settings:: - - >>> import zarr - >>> z = zarr.v2.create((10000, 10000), chunks=(1000, 1000)) - >>> z - - - Create an array with different some different configuration options:: - - >>> from numcodecs import Blosc - >>> compressor = Blosc(cname='zstd', clevel=1, shuffle=Blosc.BITSHUFFLE) - >>> z = zarr.v2.create((10000, 10000), chunks=(1000, 1000), dtype='i1', order='F', - ... compressor=compressor) - >>> z - - - To create an array with object dtype requires a filter that can handle Python object - encoding, e.g., `MsgPack` or `Pickle` from `numcodecs`:: - - >>> from numcodecs import MsgPack - >>> z = zarr.v2.create((10000, 10000), chunks=(1000, 1000), dtype=object, - ... object_codec=MsgPack()) - >>> z - - - Example with some filters, and also storing chunks separately from metadata:: - - >>> from numcodecs import Quantize, Adler32 - >>> store, chunk_store = dict(), dict() - >>> z = zarr.v2.create((10000, 10000), chunks=(1000, 1000), dtype='f8', - ... filters=[Quantize(digits=2, dtype='f8'), Adler32()], - ... store=store, chunk_store=chunk_store) - >>> z - - - """ - - # handle polymorphic store arg - store = normalize_store_arg(store, mode="w") - - # API compatibility with h5py - compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs) - - # optional array metadata - if dimension_separator is None: - dimension_separator = getattr(store, "_dimension_separator", None) - else: - store_separator = getattr(store, "_dimension_separator", None) - if store_separator not in (None, dimension_separator): - raise ValueError( - f"Specified dimension_separator: {dimension_separator}" - f"conflicts with store's separator: " - f"{store_separator}" - ) - dimension_separator = normalize_dimension_separator(dimension_separator) - - # initialize array metadata - init_array( - store, - shape=shape, - chunks=chunks, - dtype=dtype, - compressor=compressor, - fill_value=fill_value, - order=order, - overwrite=overwrite, - path=path, - chunk_store=chunk_store, - filters=filters, - object_codec=object_codec, - dimension_separator=dimension_separator, - ) - - # instantiate array - z = Array( - store, - path=path, - chunk_store=chunk_store, - synchronizer=synchronizer, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - read_only=read_only, - write_empty_chunks=write_empty_chunks, - meta_array=meta_array, - ) - - return z - - -def _kwargs_compat(compressor, fill_value, kwargs): - # to be compatible with h5py, as well as backwards-compatible with Zarr - # 1.x, accept 'compression' and 'compression_opts' keyword arguments - - if compressor != "default": - # 'compressor' overrides 'compression' - if "compression" in kwargs: - warn( - "'compression' keyword argument overridden by 'compressor'", - stacklevel=3, - ) - del kwargs["compression"] - if "compression_opts" in kwargs: - warn( - "'compression_opts' keyword argument overridden by 'compressor'", - stacklevel=3, - ) - del kwargs["compression_opts"] - - elif "compression" in kwargs: - compression = kwargs.pop("compression") - compression_opts = kwargs.pop("compression_opts", None) - - if compression is None or compression == "none": - compressor = None - - elif compression == "default": - compressor = default_compressor - - elif isinstance(compression, str): - codec_cls = codec_registry[compression] - - # handle compression_opts - if isinstance(compression_opts, dict): - compressor = codec_cls(**compression_opts) - elif isinstance(compression_opts, (list, tuple)): - compressor = codec_cls(*compression_opts) - elif compression_opts is None: - compressor = codec_cls() - else: - # assume single argument, e.g., int - compressor = codec_cls(compression_opts) - - # be lenient here if user gives compressor as 'compression' - elif hasattr(compression, "get_config"): - compressor = compression - - else: - raise ValueError("bad value for compression: %r" % compression) - - # handle 'fillvalue' - if "fillvalue" in kwargs: - # to be compatible with h5py, accept 'fillvalue' instead of - # 'fill_value' - fill_value = kwargs.pop("fillvalue") - - # ignore other keyword arguments - for k in kwargs: - warn("ignoring keyword argument %r" % k) - - return compressor, fill_value - - -def empty(shape, **kwargs): - """Create an empty array. - - For parameter definitions see :func:`zarr.v2.creation.create`. - - Notes - ----- - The contents of an empty Zarr array are not defined. On attempting to - retrieve data from an empty Zarr array, any values may be returned, - and these are not guaranteed to be stable from one access to the next. - - """ - return create(shape=shape, fill_value=None, **kwargs) - - -def zeros(shape, **kwargs): - """Create an array, with zero being used as the default value for - uninitialized portions of the array. - - For parameter definitions see :func:`zarr.v2.creation.create`. - - Examples - -------- - >>> import zarr - >>> z = zarr.v2.zeros((10000, 10000), chunks=(1000, 1000)) - >>> z - - >>> z[:2, :2] - array([[0., 0.], - [0., 0.]]) - - """ - - return create(shape=shape, fill_value=0, **kwargs) - - -def ones(shape, **kwargs): - """Create an array, with one being used as the default value for - uninitialized portions of the array. - - For parameter definitions see :func:`zarr.v2.creation.create`. - - Examples - -------- - >>> import zarr - >>> z = zarr.v2.ones((10000, 10000), chunks=(1000, 1000)) - >>> z - - >>> z[:2, :2] - array([[1., 1.], - [1., 1.]]) - - """ - - return create(shape=shape, fill_value=1, **kwargs) - - -def full(shape, fill_value, **kwargs): - """Create an array, with `fill_value` being used as the default value for - uninitialized portions of the array. - - For parameter definitions see :func:`zarr.v2.creation.create`. - - Examples - -------- - >>> import zarr - >>> z = zarr.v2.full((10000, 10000), chunks=(1000, 1000), fill_value=42) - >>> z - - >>> z[:2, :2] - array([[42., 42.], - [42., 42.]]) - - """ - - return create(shape=shape, fill_value=fill_value, **kwargs) - - -def _get_shape_chunks(a): - shape = None - chunks = None - - if hasattr(a, "shape") and isinstance(a.shape, tuple): - shape = a.shape - - if hasattr(a, "chunks") and isinstance(a.chunks, tuple) and (len(a.chunks) == len(a.shape)): - chunks = a.chunks - - elif hasattr(a, "chunklen"): - # bcolz carray - chunks = (a.chunklen,) + a.shape[1:] - - return shape, chunks - - -def array(data, **kwargs): - """Create an array filled with `data`. - - The `data` argument should be a NumPy array or array-like object. For - other parameter definitions see :func:`zarr.v2.creation.create`. - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> a = np.arange(100000000).reshape(10000, 10000) - >>> z = zarr.v2.array(a, chunks=(1000, 1000)) - >>> z - - - """ - - # ensure data is array-like - if not hasattr(data, "shape") or not hasattr(data, "dtype"): - data = np.asanyarray(data) - - # setup dtype - kw_dtype = kwargs.get("dtype") - if kw_dtype is None: - kwargs["dtype"] = data.dtype - else: - kwargs["dtype"] = kw_dtype - - # setup shape and chunks - data_shape, data_chunks = _get_shape_chunks(data) - kwargs["shape"] = data_shape - kw_chunks = kwargs.get("chunks") - if kw_chunks is None: - kwargs["chunks"] = data_chunks - else: - kwargs["chunks"] = kw_chunks - - # pop read-only to apply after storing the data - read_only = kwargs.pop("read_only", False) - - # instantiate array - z = create(**kwargs) - - # fill with data - z[...] = data - - # set read_only property afterwards - z.read_only = read_only - - return z - - -def open_array( - store=None, - mode="a", - shape=None, - chunks=True, - dtype=None, - compressor="default", - fill_value=0, - order="C", - synchronizer=None, - filters=None, - cache_metadata=True, - cache_attrs=True, - path=None, - object_codec=None, - chunk_store=None, - storage_options=None, - partial_decompress=False, - write_empty_chunks=True, - *, - dimension_separator=None, - meta_array=None, - **kwargs, -): - """Open an array using file-mode-like semantics. - - Parameters - ---------- - store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist); 'a' means read/write (create if doesn't - exist); 'w' means create (overwrite if exists); 'w-' means create - (fail if exists). - shape : int or tuple of ints, optional - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. If True, will be guessed from `shape` and `dtype`. If - False, will be set to `shape`, i.e., single chunk for the whole array. - If an int, the chunk size in each dimension will be given by the value - of `chunks`. Default is True. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object, optional - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - synchronizer : object, optional - Array synchronizer. - filters : sequence, optional - Sequence of filters to use to encode chunk data prior to compression. - cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - path : string, optional - Array path within store. - object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. - chunk_store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - storage_options : dict - If using an fsspec URL to create the store, these will be passed to - the backend implementation. Ignored otherwise. - partial_decompress : bool, optional - If True and while the chunk_store is a FSStore and the compression used - is Blosc, when getting data from the array chunks will be partially - read and decompressed when possible. - write_empty_chunks : bool, optional - If True (default), all chunks will be stored regardless of their - contents. If False, each chunk is compared to the array's fill value - prior to storing. If a chunk is uniformly equal to the fill value, then - that chunk is not be stored, and the store entry for that chunk's key - is deleted. This setting enables sparser storage, as only chunks with - non-fill-value data are stored, at the expense of overhead associated - with checking the data of each chunk. - - .. versionadded:: 2.11 - - dimension_separator : {None, '.', '/'}, optional - Can be used to specify whether the array is in a flat ('.') or nested - ('/') format. If None, the appropriate value will be read from `store` - when present. Otherwise, defaults to '.'. - meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. - - .. versionadded:: 2.15 - - Returns - ------- - z : zarr.v2.core.Array - - Examples - -------- - >>> import numpy as np - >>> import zarr - >>> z1 = zarr.v2.open_array('data/example.zarr', mode='w', shape=(10000, 10000), - ... chunks=(1000, 1000), fill_value=0) - >>> z1[:] = np.arange(100000000).reshape(10000, 10000) - >>> z1 - - >>> z2 = zarr.v2.open_array('data/example.zarr', mode='r') - >>> z2 - - >>> np.all(z1[:] == z2[:]) - True - - Notes - ----- - There is no need to close an array. Data are automatically flushed to the - file system. - - """ - - # use same mode semantics as h5py - # r : read only, must exist - # r+ : read/write, must exist - # w : create, delete if exists - # w- or x : create, fail if exists - # a : read/write if exists, create otherwise (default) - - # handle polymorphic store arg - store = normalize_store_arg(store, storage_options=storage_options, mode=mode) - - if chunk_store is not None: - chunk_store = normalize_store_arg(chunk_store, storage_options=storage_options, mode=mode) - - # respect the dimension separator specified in a store, if present - if dimension_separator is None: - if hasattr(store, "_dimension_separator"): - dimension_separator = store._dimension_separator - else: - dimension_separator = "." - - path = normalize_storage_path(path) - - # API compatibility with h5py - compressor, fill_value = _kwargs_compat(compressor, fill_value, kwargs) - - # ensure fill_value of correct type - if fill_value is not None: - fill_value = np.array(fill_value, dtype=dtype)[()] - - # ensure store is initialized - - if mode in ["r", "r+"]: - if not contains_array(store, path=path): - if contains_group(store, path=path): - raise ContainsGroupError(path) - raise ArrayNotFoundError(path) - - elif mode == "w": - init_array( - store, - shape=shape, - chunks=chunks, - dtype=dtype, - compressor=compressor, - fill_value=fill_value, - order=order, - filters=filters, - overwrite=True, - path=path, - object_codec=object_codec, - chunk_store=chunk_store, - dimension_separator=dimension_separator, - ) - - elif mode == "a": - if not contains_array(store, path=path): - if contains_group(store, path=path): - raise ContainsGroupError(path) - init_array( - store, - shape=shape, - chunks=chunks, - dtype=dtype, - compressor=compressor, - fill_value=fill_value, - order=order, - filters=filters, - path=path, - object_codec=object_codec, - chunk_store=chunk_store, - dimension_separator=dimension_separator, - ) - - elif mode in ["w-", "x"]: - if contains_group(store, path=path): - raise ContainsGroupError(path) - elif contains_array(store, path=path): - raise ContainsArrayError(path) - else: - init_array( - store, - shape=shape, - chunks=chunks, - dtype=dtype, - compressor=compressor, - fill_value=fill_value, - order=order, - filters=filters, - path=path, - object_codec=object_codec, - chunk_store=chunk_store, - dimension_separator=dimension_separator, - ) - - # determine read only status - read_only = mode == "r" - - # instantiate array - z = Array( - store, - read_only=read_only, - synchronizer=synchronizer, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - path=path, - chunk_store=chunk_store, - write_empty_chunks=write_empty_chunks, - meta_array=meta_array, - ) - - return z - - -def _like_args(a, kwargs): - shape, chunks = _get_shape_chunks(a) - if shape is not None: - kwargs.setdefault("shape", shape) - if chunks is not None: - kwargs.setdefault("chunks", chunks) - - if hasattr(a, "dtype"): - kwargs.setdefault("dtype", a.dtype) - - if isinstance(a, Array): - kwargs.setdefault("compressor", a.compressor) - kwargs.setdefault("order", a.order) - kwargs.setdefault("filters", a.filters) - else: - kwargs.setdefault("compressor", "default") - kwargs.setdefault("order", "C") - - -def empty_like(a, **kwargs): - """Create an empty array like `a`.""" - _like_args(a, kwargs) - return empty(**kwargs) - - -def zeros_like(a, **kwargs): - """Create an array of zeros like `a`.""" - _like_args(a, kwargs) - return zeros(**kwargs) - - -def ones_like(a, **kwargs): - """Create an array of ones like `a`.""" - _like_args(a, kwargs) - return ones(**kwargs) - - -def full_like(a, **kwargs): - """Create a filled array like `a`.""" - _like_args(a, kwargs) - if isinstance(a, Array): - kwargs.setdefault("fill_value", a.fill_value) - return full(**kwargs) - - -def open_like(a, path, **kwargs): - """Open a persistent array like `a`.""" - _like_args(a, kwargs) - if isinstance(a, Array): - kwargs.setdefault("fill_value", a.fill_value) - return open_array(path, **kwargs) diff --git a/src/zarr/v2/errors.py b/src/zarr/v2/errors.py deleted file mode 100644 index 30c9b13d39..0000000000 --- a/src/zarr/v2/errors.py +++ /dev/null @@ -1,80 +0,0 @@ -class MetadataError(Exception): - pass - - -class CopyError(RuntimeError): - pass - - -class _BaseZarrError(ValueError): - _msg = "" - - def __init__(self, *args): - super().__init__(self._msg.format(*args)) - - -class ArrayIndexError(IndexError): - pass - - -class _BaseZarrIndexError(IndexError): - _msg = "" - - def __init__(self, *args): - super().__init__(self._msg.format(*args)) - - -class ContainsGroupError(_BaseZarrError): - _msg = "path {0!r} contains a group" - - -class ContainsArrayError(_BaseZarrError): - _msg = "path {0!r} contains an array" - - -class ArrayNotFoundError(_BaseZarrError): - _msg = "array not found at path %r' {0!r}" - - -class GroupNotFoundError(_BaseZarrError): - _msg = "group not found at path {0!r}" - - -class PathNotFoundError(_BaseZarrError): - _msg = "nothing found at path {0!r}" - - -class BadCompressorError(_BaseZarrError): - _msg = "bad compressor; expected Codec object, found {0!r}" - - -class FSPathExistNotDir(GroupNotFoundError): - _msg = "path exists but is not a directory: %r" - - -class ReadOnlyError(PermissionError): - def __init__(self): - super().__init__("object is read-only") - - -class BoundsCheckError(_BaseZarrIndexError): - _msg = "index out of bounds for dimension with length {0}" - - -class NegativeStepError(IndexError): - def __init__(self): - super().__init__("only slices with step >= 1 are supported") - - -def err_too_many_indices(selection, shape): - raise IndexError( - "too many indices for array; expected {}, got {}".format(len(shape), len(selection)) - ) - - -class VindexInvalidSelectionError(_BaseZarrIndexError): - _msg = ( - "unsupported selection type for vectorized indexing; only " - "coordinate selection (tuple of integer arrays) and mask selection " - "(single Boolean array) are supported; got {0!r}" - ) diff --git a/src/zarr/v2/hierarchy.py b/src/zarr/v2/hierarchy.py deleted file mode 100644 index 25e47311b6..0000000000 --- a/src/zarr/v2/hierarchy.py +++ /dev/null @@ -1,1401 +0,0 @@ -from collections.abc import MutableMapping -from itertools import islice -from typing import Any - -import numpy as np - -from zarr.v2.attrs import Attributes -from zarr.v2.core import Array -from zarr.v2.creation import ( - array, - create, - empty, - empty_like, - full, - full_like, - ones, - ones_like, - zeros, - zeros_like, -) -from zarr.v2.errors import ( - ContainsArrayError, - ContainsGroupError, - GroupNotFoundError, - ReadOnlyError, -) -from zarr.v2.storage import ( - _prefix_to_group_key, - BaseStore, - MemoryStore, - group_meta_key, - attrs_key, - contains_array, - contains_group, - init_group, - listdir, - normalize_store_arg, - rename, - rmdir, -) - -from zarr.v2.util import ( - InfoReporter, - TreeViewer, - is_valid_python_name, - nolock, - normalize_shape, - normalize_storage_path, -) - - -class Group(MutableMapping[str, Any]): - """Instantiate a group from an initialized store. - - Parameters - ---------- - store : MutableMapping - Group store, already initialized. - If the Group is used in a context manager, and the store has a ``close`` method, - it will be called on exit. - path : string, optional - Group path. - read_only : bool, optional - True if group should be protected against modification. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - synchronizer : object, optional - Array synchronizer. - - meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. - - .. versionadded:: 2.13 - - Attributes - ---------- - store - path - name - read_only - chunk_store - synchronizer - attrs - info - meta_array - - Methods - ------- - __len__ - __iter__ - __contains__ - __getitem__ - __enter__ - __exit__ - group_keys - groups - array_keys - arrays - visit - visitkeys - visitvalues - visititems - tree - create_group - require_group - create_groups - require_groups - create_dataset - require_dataset - create - empty - zeros - ones - full - array - empty_like - zeros_like - ones_like - full_like - info - move - - """ - - def __init__( - self, - store, - path=None, - read_only=False, - chunk_store=None, - cache_attrs=True, - synchronizer=None, - *, - meta_array=None, - ): - store: BaseStore = _normalize_store_arg(store) - if chunk_store is not None: - chunk_store: BaseStore = _normalize_store_arg(chunk_store) - self._store = store - self._chunk_store = chunk_store - self._path = normalize_storage_path(path) - if self._path: - self._key_prefix = self._path + "/" - else: - self._key_prefix = "" - self._read_only = read_only - self._synchronizer = synchronizer - if meta_array is not None: - self._meta_array = np.empty_like(meta_array, shape=()) - else: - self._meta_array = np.empty(()) - - # guard conditions - if contains_array(store, path=self._path): - raise ContainsArrayError(path) - - # initialize metadata - mkey = None - try: - mkey = _prefix_to_group_key(self._store, self._key_prefix) - assert not mkey.endswith("root/.group") - meta_bytes = store[mkey] - except KeyError: - raise GroupNotFoundError(path) - else: - self._meta = self._store._metadata_class.decode_group_metadata(meta_bytes) - - # setup attributes - akey = self._key_prefix + attrs_key - - self._attrs = Attributes( - store, key=akey, read_only=read_only, cache=cache_attrs, synchronizer=synchronizer - ) - - # setup info - self._info = InfoReporter(self) - - @property - def store(self): - """A MutableMapping providing the underlying storage for the group.""" - return self._store - - @property - def path(self): - """Storage path.""" - return self._path - - @property - def name(self): - """Group name following h5py convention.""" - if self._path: - # follow h5py convention: add leading slash - name = self._path - if name[0] != "/": - name = "/" + name - return name - return "/" - - @property - def basename(self): - """Final component of name.""" - return self.name.split("/")[-1] - - @property - def read_only(self): - """A boolean, True if modification operations are not permitted.""" - return self._read_only - - @property - def chunk_store(self): - """A MutableMapping providing the underlying storage for array chunks.""" - if self._chunk_store is None: - return self._store - else: - return self._chunk_store - - @property - def synchronizer(self): - """Object used to synchronize write access to groups and arrays.""" - return self._synchronizer - - @property - def attrs(self): - """A MutableMapping containing user-defined attributes. Note that - attribute values must be JSON serializable.""" - return self._attrs - - @property - def info(self): - """Return diagnostic information about the group.""" - return self._info - - @property - def meta_array(self): - """An array-like instance to use for determining arrays to create and return - to users. - """ - return self._meta_array - - def __eq__(self, other): - return ( - isinstance(other, Group) - and self._store == other.store - and self._read_only == other.read_only - and self._path == other.path - # N.B., no need to compare attributes, should be covered by - # store comparison - ) - - def __iter__(self): - """Return an iterator over group member names. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> for name in g1: - ... print(name) - bar - baz - foo - quux - - """ - - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if contains_array(self._store, path) or contains_group(self._store, path): - yield key - - def __len__(self): - """Number of members.""" - return sum(1 for _ in self) - - def __repr__(self): - t = type(self) - r = "<{}.{}".format(t.__module__, t.__name__) - if self.name: - r += " %r" % self.name - if self._read_only: - r += " read-only" - r += ">" - return r - - def __enter__(self): - """Return the Group for use as a context manager.""" - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """Call the close method of the underlying Store.""" - self.store.close() - - def info_items(self): - def typestr(o): - return "{}.{}".format(type(o).__module__, type(o).__name__) - - items = [] - - # basic info - if self.name is not None: - items += [("Name", self.name)] - items += [ - ("Type", typestr(self)), - ("Read-only", str(self.read_only)), - ] - - # synchronizer - if self._synchronizer is not None: - items += [("Synchronizer type", typestr(self._synchronizer))] - - # storage info - items += [("Store type", typestr(self._store))] - if self._chunk_store is not None: - items += [("Chunk store type", typestr(self._chunk_store))] - - # members - items += [("No. members", len(self))] - array_keys = sorted(self.array_keys()) - group_keys = sorted(self.group_keys()) - items += [("No. arrays", len(array_keys))] - items += [("No. groups", len(group_keys))] - if array_keys: - items += [("Arrays", ", ".join(array_keys))] - if group_keys: - items += [("Groups", ", ".join(group_keys))] - - return items - - def __getstate__(self): - return { - "store": self._store, - "path": self._path, - "read_only": self._read_only, - "chunk_store": self._chunk_store, - "cache_attrs": self._attrs.cache, - "synchronizer": self._synchronizer, - "meta_array": self._meta_array, - } - - def __setstate__(self, state): - self.__init__(**state) - - def _item_path(self, item): - absolute = isinstance(item, str) and item and item[0] == "/" - path = normalize_storage_path(item) - if not absolute and self._path: - path = self._key_prefix + path - return path - - def __contains__(self, item): - """Test for group membership. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> d1 = g1.create_dataset('bar', shape=100, chunks=10) - >>> 'foo' in g1 - True - >>> 'bar' in g1 - True - >>> 'baz' in g1 - False - - """ - path = self._item_path(item) - return contains_array(self._store, path) or contains_group( - self._store, path, explicit_only=False - ) - - def __getitem__(self, item): - """Obtain a group member. - - Parameters - ---------- - item : string - Member name or path. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> d1 = g1.create_dataset('foo/bar/baz', shape=100, chunks=10) - >>> g1['foo'] - - >>> g1['foo/bar'] - - >>> g1['foo/bar/baz'] - - - """ - path = self._item_path(item) - if contains_array(self._store, path): - return Array( - self._store, - read_only=self._read_only, - path=path, - chunk_store=self._chunk_store, - synchronizer=self._synchronizer, - cache_attrs=self.attrs.cache, - meta_array=self._meta_array, - ) - elif contains_group(self._store, path, explicit_only=True): - return Group( - self._store, - read_only=self._read_only, - path=path, - chunk_store=self._chunk_store, - cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer, - meta_array=self._meta_array, - ) - else: - raise KeyError(item) - - def __setitem__(self, item, value): - self.array(item, value, overwrite=True) - - def __delitem__(self, item): - return self._write_op(self._delitem_nosync, item) - - def _delitem_nosync(self, item): - path = self._item_path(item) - if contains_array(self._store, path) or contains_group( - self._store, path, explicit_only=False - ): - rmdir(self._store, path) - else: - raise KeyError(item) - - def __getattr__(self, item): - # allow access to group members via dot notation - try: - return self.__getitem__(item) - except KeyError: - raise AttributeError - - def __dir__(self): - # noinspection PyUnresolvedReferences - base = super().__dir__() - keys = sorted(set(base + list(self))) - keys = [k for k in keys if is_valid_python_name(k)] - return keys - - def _ipython_key_completions_(self): - return sorted(self) - - def group_keys(self): - """Return an iterator over member names for groups only. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> sorted(g1.group_keys()) - ['bar', 'foo'] - - """ - - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if contains_group(self._store, path): - yield key - - def groups(self): - """Return an iterator over (name, value) pairs for groups only. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> for n, v in g1.groups(): - ... print(n, type(v)) - bar - foo - - """ - - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if contains_group(self._store, path, explicit_only=False): - yield ( - key, - Group( - self._store, - path=path, - read_only=self._read_only, - chunk_store=self._chunk_store, - cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer, - ), - ) - - def array_keys(self, recurse=False): - """Return an iterator over member names for arrays only. - - Parameters - ---------- - recurse : recurse, optional - Option to return member names for all arrays, even from groups - below the current one. If False, only member names for arrays in - the current group will be returned. Default value is False. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> sorted(g1.array_keys()) - ['baz', 'quux'] - - """ - return self._array_iter(keys_only=True, method="array_keys", recurse=recurse) - - def arrays(self, recurse=False): - """Return an iterator over (name, value) pairs for arrays only. - - Parameters - ---------- - recurse : recurse, optional - Option to return (name, value) pairs for all arrays, even from groups - below the current one. If False, only (name, value) pairs for arrays in - the current group will be returned. Default value is False. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> d1 = g1.create_dataset('baz', shape=100, chunks=10) - >>> d2 = g1.create_dataset('quux', shape=200, chunks=20) - >>> for n, v in g1.arrays(): - ... print(n, type(v)) - baz - quux - - """ - return self._array_iter(keys_only=False, method="arrays", recurse=recurse) - - def _array_iter(self, keys_only, method, recurse): - for key in sorted(listdir(self._store, self._path)): - path = self._key_prefix + key - if contains_array(self._store, path): - _key = key.rstrip("/") - yield _key if keys_only else (_key, self[key]) - elif recurse and contains_group(self._store, path): - group = self[key] - yield from getattr(group, method)(recurse=recurse) - - def visitvalues(self, func): - """Run ``func`` on each object. - - Note: If ``func`` returns ``None`` (or doesn't return), - iteration continues. However, if ``func`` returns - anything else, it ceases and returns that value. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> def print_visitor(obj): - ... print(obj) - >>> g1.visitvalues(print_visitor) - - - - - >>> g3.visitvalues(print_visitor) - - - - """ - - def _visit(obj): - yield obj - keys = sorted(getattr(obj, "keys", lambda: [])()) - for k in keys: - yield from _visit(obj[k]) - - for each_obj in islice(_visit(self), 1, None): - value = func(each_obj) - if value is not None: - return value - - def visit(self, func): - """Run ``func`` on each object's path. - - Note: If ``func`` returns ``None`` (or doesn't return), - iteration continues. However, if ``func`` returns - anything else, it ceases and returns that value. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> def print_visitor(name): - ... print(name) - >>> g1.visit(print_visitor) - bar - bar/baz - bar/quux - foo - >>> g3.visit(print_visitor) - baz - quux - - Search for members matching some name query can be implemented using - ``visit`` that is, ``find`` and ``findall``. Consider the following - tree:: - - / - ├── aaa - │ └── bbb - │ └── ccc - │ └── aaa - ├── bar - └── foo - - It is created as follows: - - >>> root = zarr.v2.group() - >>> foo = root.create_group("foo") - >>> bar = root.create_group("bar") - >>> root.create_group("aaa").create_group("bbb").create_group("ccc").create_group("aaa") - - - For ``find``, the first path that matches a given pattern (for example - "aaa") is returned. Note that a non-None value is returned in the visit - function to stop further iteration. - - >>> import re - >>> pattern = re.compile("aaa") - >>> found = None - >>> def find(path): - ... global found - ... if pattern.search(path) is not None: - ... found = path - ... return True - ... - >>> root.visit(find) - True - >>> print(found) - aaa - - For ``findall``, all the results are gathered into a list - - >>> pattern = re.compile("aaa") - >>> found = [] - >>> def findall(path): - ... if pattern.search(path) is not None: - ... found.append(path) - ... - >>> root.visit(findall) - >>> print(found) - ['aaa', 'aaa/bbb', 'aaa/bbb/ccc', 'aaa/bbb/ccc/aaa'] - - To match only on the last part of the path, use a greedy regex to filter - out the prefix: - - >>> prefix_pattern = re.compile(r".*/") - >>> pattern = re.compile("aaa") - >>> found = [] - >>> def findall(path): - ... match = prefix_pattern.match(path) - ... if match is None: - ... name = path - ... else: - ... _, end = match.span() - ... name = path[end:] - ... if pattern.search(name) is not None: - ... found.append(path) - ... return None - ... - >>> root.visit(findall) - >>> print(found) - ['aaa', 'aaa/bbb/ccc/aaa'] - """ - - base_len = len(self.name) - return self.visitvalues(lambda o: func(o.name[base_len:].lstrip("/"))) - - def visitkeys(self, func): - """An alias for :py:meth:`~Group.visit`.""" - - return self.visit(func) - - def visititems(self, func): - """Run ``func`` on each object's path and the object itself. - - Note: If ``func`` returns ``None`` (or doesn't return), - iteration continues. However, if ``func`` returns - anything else, it ceases and returns that value. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> def print_visitor(name, obj): - ... print((name, obj)) - >>> g1.visititems(print_visitor) - ('bar', ) - ('bar/baz', ) - ('bar/quux', ) - ('foo', ) - >>> g3.visititems(print_visitor) - ('baz', ) - ('quux', ) - - """ - - base_len = len(self.name) - return self.visitvalues(lambda o: func(o.name[base_len:].lstrip("/"), o)) - - def tree(self, expand=False, level=None): - """Provide a ``print``-able display of the hierarchy. - - Parameters - ---------- - expand : bool, optional - Only relevant for HTML representation. If True, tree will be fully expanded. - level : int, optional - Maximum depth to descend into hierarchy. - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g3.create_group('baz') - >>> g5 = g3.create_group('quux') - >>> d1 = g5.create_dataset('baz', shape=100, chunks=10) - >>> g1.tree() - / - ├── bar - │ ├── baz - │ └── quux - │ └── baz (100,) float64 - └── foo - >>> g1.tree(level=2) - / - ├── bar - │ ├── baz - │ └── quux - └── foo - >>> g3.tree() - bar - ├── baz - └── quux - └── baz (100,) float64 - - Notes - ----- - Please note that this is an experimental feature. The behaviour of this - function is still evolving and the default output and/or parameters may change - in future versions. - - """ - - return TreeViewer(self, expand=expand, level=level) - - def _write_op(self, f, *args, **kwargs): - # guard condition - if self._read_only: - raise ReadOnlyError - - if self._synchronizer is None: - # no synchronization - lock = nolock - else: - # synchronize on the root group - lock = self._synchronizer[group_meta_key] - - with lock: - return f(*args, **kwargs) - - def create_group(self, name, overwrite=False): - """Create a sub-group. - - Parameters - ---------- - name : string - Group name. - overwrite : bool, optional - If True, overwrite any existing array with the given name. - - Returns - ------- - g : zarr.v2.hierarchy.Group - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.create_group('foo') - >>> g3 = g1.create_group('bar') - >>> g4 = g1.create_group('baz/quux') - - """ - - return self._write_op(self._create_group_nosync, name, overwrite=overwrite) - - def _create_group_nosync(self, name, overwrite=False): - path = self._item_path(name) - - # create terminal group - init_group(self._store, path=path, chunk_store=self._chunk_store, overwrite=overwrite) - - return Group( - self._store, - path=path, - read_only=self._read_only, - chunk_store=self._chunk_store, - cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer, - ) - - def create_groups(self, *names, **kwargs): - """Convenience method to create multiple groups in a single call.""" - return tuple(self.create_group(name, **kwargs) for name in names) - - def require_group(self, name, overwrite=False): - """Obtain a sub-group, creating one if it doesn't exist. - - Parameters - ---------- - name : string - Group name. - overwrite : bool, optional - Overwrite any existing array with given `name` if present. - - Returns - ------- - g : zarr.v2.hierarchy.Group - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> g2 = g1.require_group('foo') - >>> g3 = g1.require_group('foo') - >>> g2 == g3 - True - - """ - - return self._write_op(self._require_group_nosync, name, overwrite=overwrite) - - def _require_group_nosync(self, name, overwrite=False): - path = self._item_path(name) - - # create terminal group if necessary - if not contains_group(self._store, path): - init_group( - store=self._store, path=path, chunk_store=self._chunk_store, overwrite=overwrite - ) - - return Group( - self._store, - path=path, - read_only=self._read_only, - chunk_store=self._chunk_store, - cache_attrs=self.attrs.cache, - synchronizer=self._synchronizer, - ) - - def require_groups(self, *names): - """Convenience method to require multiple groups in a single call.""" - return tuple(self.require_group(name) for name in names) - - # noinspection PyIncorrectDocstring - def create_dataset(self, name, **kwargs): - """Create an array. - - Arrays are known as "datasets" in HDF5 terminology. For compatibility - with h5py, Zarr groups also implement the require_dataset() method. - - Parameters - ---------- - name : string - Array name. - data : array-like, optional - Initial data. - shape : int or tuple of ints - Array shape. - chunks : int or tuple of ints, optional - Chunk shape. If not provided, will be guessed from `shape` and - `dtype`. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - synchronizer : zarr.v2.sync.ArraySynchronizer, optional - Array synchronizer. - filters : sequence of Codecs, optional - Sequence of filters to use to encode chunk data prior to - compression. - overwrite : bool, optional - If True, replace any existing array or group with the given name. - cache_metadata : bool, optional - If True, array configuration metadata will be cached for the - lifetime of the object. If False, array metadata will be reloaded - prior to all data access and modification operations (may incur - overhead depending on storage and data access pattern). - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Returns - ------- - a : zarr.v2.core.Array - - Examples - -------- - >>> import zarr - >>> g1 = zarr.v2.group() - >>> d1 = g1.create_dataset('foo', shape=(10000, 10000), - ... chunks=(1000, 1000)) - >>> d1 - - >>> d2 = g1.create_dataset('bar/baz/qux', shape=(100, 100, 100), - ... chunks=(100, 10, 10)) - >>> d2 - - - """ - assert "mode" not in kwargs - - return self._write_op(self._create_dataset_nosync, name, **kwargs) - - def _create_dataset_nosync(self, name, data=None, **kwargs): - assert "mode" not in kwargs - path = self._item_path(name) - - # determine synchronizer - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - - # create array - if data is None: - a = create(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - else: - a = array(data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - return a - - def require_dataset(self, name, shape, dtype=None, exact=False, **kwargs): - """Obtain an array, creating if it doesn't exist. - - Arrays are known as "datasets" in HDF5 terminology. For compatibility - with h5py, Zarr groups also implement the create_dataset() method. - - Other `kwargs` are as per :func:`zarr.v2.hierarchy.Group.create_dataset`. - - Parameters - ---------- - name : string - Array name. - shape : int or tuple of ints - Array shape. - dtype : string or dtype, optional - NumPy dtype. - exact : bool, optional - If True, require `dtype` to match exactly. If false, require - `dtype` can be cast from array dtype. - - """ - - return self._write_op( - self._require_dataset_nosync, name, shape=shape, dtype=dtype, exact=exact, **kwargs - ) - - def _require_dataset_nosync(self, name, shape, dtype=None, exact=False, **kwargs): - path = self._item_path(name) - - if contains_array(self._store, path): - # array already exists at path, validate that it is the right shape and type - - synchronizer = kwargs.get("synchronizer", self._synchronizer) - cache_metadata = kwargs.get("cache_metadata", True) - cache_attrs = kwargs.get("cache_attrs", self.attrs.cache) - a = Array( - self._store, - path=path, - read_only=self._read_only, - chunk_store=self._chunk_store, - synchronizer=synchronizer, - cache_metadata=cache_metadata, - cache_attrs=cache_attrs, - meta_array=self._meta_array, - ) - shape = normalize_shape(shape) - if shape != a.shape: - raise TypeError( - "shape do not match existing array; expected {}, got {}".format(a.shape, shape) - ) - dtype = np.dtype(dtype) - if exact: - if dtype != a.dtype: - raise TypeError( - "dtypes do not match exactly; expected {}, got {}".format(a.dtype, dtype) - ) - else: - if not np.can_cast(dtype, a.dtype): - raise TypeError("dtypes ({}, {}) cannot be safely cast".format(dtype, a.dtype)) - return a - - else: - return self._create_dataset_nosync(name, shape=shape, dtype=dtype, **kwargs) - - def create(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.create`.""" - return self._write_op(self._create_nosync, name, **kwargs) - - def _create_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return create(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - def empty(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.empty`.""" - return self._write_op(self._empty_nosync, name, **kwargs) - - def _empty_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return empty(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - def zeros(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.zeros`.""" - return self._write_op(self._zeros_nosync, name, **kwargs) - - def _zeros_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return zeros(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - def ones(self, name, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.ones`.""" - return self._write_op(self._ones_nosync, name, **kwargs) - - def _ones_nosync(self, name, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return ones(store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - def full(self, name, fill_value, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.full`.""" - return self._write_op(self._full_nosync, name, fill_value, **kwargs) - - def _full_nosync(self, name, fill_value, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return full( - store=self._store, - path=path, - chunk_store=self._chunk_store, - fill_value=fill_value, - **kwargs, - ) - - def array(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.array`.""" - return self._write_op(self._array_nosync, name, data, **kwargs) - - def _array_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return array(data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs) - - def empty_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.empty_like`.""" - return self._write_op(self._empty_like_nosync, name, data, **kwargs) - - def _empty_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return empty_like( - data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs - ) - - def zeros_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.zeros_like`.""" - return self._write_op(self._zeros_like_nosync, name, data, **kwargs) - - def _zeros_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return zeros_like( - data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs - ) - - def ones_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.ones_like`.""" - return self._write_op(self._ones_like_nosync, name, data, **kwargs) - - def _ones_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return ones_like( - data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs - ) - - def full_like(self, name, data, **kwargs): - """Create an array. Keyword arguments as per - :func:`zarr.v2.creation.full_like`.""" - return self._write_op(self._full_like_nosync, name, data, **kwargs) - - def _full_like_nosync(self, name, data, **kwargs): - path = self._item_path(name) - kwargs.setdefault("synchronizer", self._synchronizer) - kwargs.setdefault("cache_attrs", self.attrs.cache) - return full_like( - data, store=self._store, path=path, chunk_store=self._chunk_store, **kwargs - ) - - def _move_nosync(self, path, new_path): - rename(self._store, path, new_path) - if self._chunk_store is not None: - rename(self._chunk_store, path, new_path) - - def move(self, source, dest): - """Move contents from one path to another relative to the Group. - - Parameters - ---------- - source : string - Name or path to a Zarr object to move. - dest : string - New name or path of the Zarr object. - """ - - source = self._item_path(source) - dest = self._item_path(dest) - - # Check that source exists. - if not ( - contains_array(self._store, source) - or contains_group(self._store, source, explicit_only=False) - ): - raise ValueError(f'The source, "{source}", does not exist.') - if contains_array(self._store, dest) or contains_group( - self._store, dest, explicit_only=False - ): - raise ValueError(f'The dest, "{dest}", already exists.') - - # Ensure groups needed for `dest` exist. - if "/" in dest: - self.require_group("/" + dest.rsplit("/", 1)[0]) - - self._write_op(self._move_nosync, source, dest) - - -def _normalize_store_arg(store, *, storage_options=None, mode="r"): - if store is None: - return MemoryStore() - return normalize_store_arg(store, storage_options=storage_options, mode=mode) - - -def group( - store=None, - overwrite=False, - chunk_store=None, - cache_attrs=True, - synchronizer=None, - path=None, - *, - meta_array=None, -): - """Create a group. - - Parameters - ---------- - store : MutableMapping or string, optional - Store or path to directory in file system. - overwrite : bool, optional - If True, delete any pre-existing data in `store` at `path` before - creating the group. - chunk_store : MutableMapping, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - synchronizer : object, optional - Array synchronizer. - path : string, optional - Group path within store. - meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. - - .. versionadded:: 2.16.1 - - Returns - ------- - g : zarr.v2.hierarchy.Group - - Examples - -------- - Create a group in memory:: - - >>> import zarr - >>> g = zarr.v2.group() - >>> g - - - Create a group with a different store:: - - >>> store = zarr.v2.DirectoryStore('data/example.zarr') - >>> g = zarr.v2.group(store=store, overwrite=True) - >>> g - - - """ - - # handle polymorphic store arg - store = _normalize_store_arg(store, mode="w") - - path = normalize_storage_path(path) - - requires_init = overwrite or not contains_group(store) - - if requires_init: - init_group(store, overwrite=overwrite, chunk_store=chunk_store, path=path) - - return Group( - store, - read_only=False, - chunk_store=chunk_store, - cache_attrs=cache_attrs, - synchronizer=synchronizer, - path=path, - meta_array=meta_array, - ) - - -def open_group( - store=None, - mode="a", - cache_attrs=True, - synchronizer=None, - path=None, - chunk_store=None, - storage_options=None, - *, - meta_array=None, -): - """Open a group using file-mode-like semantics. - - Parameters - ---------- - store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - mode : {'r', 'r+', 'a', 'w', 'w-'}, optional - Persistence mode: 'r' means read only (must exist); 'r+' means - read/write (must exist); 'a' means read/write (create if doesn't - exist); 'w' means create (overwrite if exists); 'w-' means create - (fail if exists). - cache_attrs : bool, optional - If True (default), user attributes will be cached for attribute read - operations. If False, user attributes are reloaded from the store prior - to all attribute read operations. - synchronizer : object, optional - Array synchronizer. - path : string, optional - Group path within store. - chunk_store : MutableMapping or string, optional - Store or path to directory in file system or name of zip file. - storage_options : dict - If using an fsspec URL to create the store, these will be passed to - the backend implementation. Ignored otherwise. - meta_array : array-like, optional - An array instance to use for determining arrays to create and return - to users. Use `numpy.empty(())` by default. - - .. versionadded:: 2.13 - - Returns - ------- - g : zarr.v2.hierarchy.Group - - Examples - -------- - >>> import zarr - >>> root = zarr.v2.open_group('data/example.zarr', mode='w') - >>> foo = root.create_group('foo') - >>> bar = root.create_group('bar') - >>> root - - >>> root2 = zarr.v2.open_group('data/example.zarr', mode='a') - >>> root2 - - >>> root == root2 - True - - """ - - # handle polymorphic store arg - store = _normalize_store_arg(store, storage_options=storage_options, mode=mode) - - if chunk_store is not None: - chunk_store = _normalize_store_arg(chunk_store, storage_options=storage_options, mode=mode) - - path = normalize_storage_path(path) - - # ensure store is initialized - - if mode in ["r", "r+"]: - if not contains_group(store, path=path): - if contains_array(store, path=path): - raise ContainsArrayError(path) - raise GroupNotFoundError(path) - - elif mode == "w": - init_group(store, overwrite=True, path=path, chunk_store=chunk_store) - - elif mode == "a": - if not contains_group(store, path=path): - if contains_array(store, path=path): - raise ContainsArrayError(path) - init_group(store, path=path, chunk_store=chunk_store) - - elif mode in ["w-", "x"]: - if contains_array(store, path=path): - raise ContainsArrayError(path) - elif contains_group(store, path=path): - raise ContainsGroupError(path) - else: - init_group(store, path=path, chunk_store=chunk_store) - - # determine read only status - read_only = mode == "r" - - return Group( - store, - read_only=read_only, - cache_attrs=cache_attrs, - synchronizer=synchronizer, - path=path, - chunk_store=chunk_store, - meta_array=meta_array, - ) diff --git a/src/zarr/v2/indexing.py b/src/zarr/v2/indexing.py deleted file mode 100644 index 880baf3f72..0000000000 --- a/src/zarr/v2/indexing.py +++ /dev/null @@ -1,1074 +0,0 @@ -import collections -import itertools -import math -import numbers - -import numpy as np - - -from zarr.v2.errors import ( - ArrayIndexError, - NegativeStepError, - err_too_many_indices, - VindexInvalidSelectionError, - BoundsCheckError, -) - - -def is_integer(x): - """True if x is an integer (both pure Python or NumPy). - - Note that Python's bool is considered an integer too. - """ - return isinstance(x, numbers.Integral) - - -def is_integer_list(x): - """True if x is a list of integers. - - This function assumes ie *does not check* that all elements of the list - have the same type. Mixed type lists will result in other errors that will - bubble up anyway. - """ - return isinstance(x, list) and len(x) > 0 and is_integer(x[0]) - - -def is_integer_array(x, ndim=None): - t = not np.isscalar(x) and hasattr(x, "shape") and hasattr(x, "dtype") and x.dtype.kind in "ui" - if ndim is not None: - t = t and len(x.shape) == ndim - return t - - -def is_bool_array(x, ndim=None): - t = hasattr(x, "shape") and hasattr(x, "dtype") and x.dtype == bool - if ndim is not None: - t = t and len(x.shape) == ndim - return t - - -def is_scalar(value, dtype): - if np.isscalar(value): - return True - if isinstance(value, tuple) and dtype.names and len(value) == len(dtype.names): - return True - return False - - -def is_pure_fancy_indexing(selection, ndim): - """Check whether a selection contains only scalars or integer array-likes. - - Parameters - ---------- - selection : tuple, slice, or scalar - A valid selection value for indexing into arrays. - - Returns - ------- - is_pure : bool - True if the selection is a pure fancy indexing expression (ie not mixed - with boolean or slices). - """ - if ndim == 1: - if is_integer_list(selection) or is_integer_array(selection): - return True - # if not, we go through the normal path below, because a 1-tuple - # of integers is also allowed. - no_slicing = ( - isinstance(selection, tuple) - and len(selection) == ndim - and not (any(isinstance(elem, slice) or elem is Ellipsis for elem in selection)) - ) - return ( - no_slicing - and all( - is_integer(elem) or is_integer_list(elem) or is_integer_array(elem) - for elem in selection - ) - and any(is_integer_list(elem) or is_integer_array(elem) for elem in selection) - ) - - -def is_pure_orthogonal_indexing(selection, ndim): - if not ndim: - return False - - # Case 1: Selection is a single iterable of integers - if is_integer_list(selection) or is_integer_array(selection, ndim=1): - return True - - # Case two: selection contains either zero or one integer iterables. - # All other selection elements are slices or integers - return ( - isinstance(selection, tuple) - and len(selection) == ndim - and sum(is_integer_list(elem) or is_integer_array(elem) for elem in selection) <= 1 - and all( - is_integer_list(elem) or is_integer_array(elem) or isinstance(elem, (int, slice)) - for elem in selection - ) - ) - - -def normalize_integer_selection(dim_sel, dim_len): - # normalize type to int - dim_sel = int(dim_sel) - - # handle wraparound - if dim_sel < 0: - dim_sel = dim_len + dim_sel - - # handle out of bounds - if dim_sel >= dim_len or dim_sel < 0: - raise BoundsCheckError(dim_len) - - return dim_sel - - -ChunkDimProjection = collections.namedtuple( - "ChunkDimProjection", ("dim_chunk_ix", "dim_chunk_sel", "dim_out_sel") -) -"""A mapping from chunk to output array for a single dimension. - -Parameters ----------- -dim_chunk_ix - Index of chunk. -dim_chunk_sel - Selection of items from chunk array. -dim_out_sel - Selection of items in target (output) array. - -""" - - -class IntDimIndexer: - def __init__(self, dim_sel, dim_len, dim_chunk_len): - # normalize - dim_sel = normalize_integer_selection(dim_sel, dim_len) - - # store attributes - self.dim_sel = dim_sel - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nitems = 1 - - def __iter__(self): - dim_chunk_ix = self.dim_sel // self.dim_chunk_len - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_chunk_sel = self.dim_sel - dim_offset - dim_out_sel = None - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -def ceildiv(a, b): - return math.ceil(a / b) - - -class SliceDimIndexer: - def __init__(self, dim_sel, dim_len, dim_chunk_len): - # normalize - self.start, self.stop, self.step = dim_sel.indices(dim_len) - if self.step < 1: - raise NegativeStepError - - # store attributes - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nitems = max(0, ceildiv((self.stop - self.start), self.step)) - self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) - - def __iter__(self): - # figure out the range of chunks we need to visit - dim_chunk_ix_from = self.start // self.dim_chunk_len - dim_chunk_ix_to = ceildiv(self.stop, self.dim_chunk_len) - - # iterate over chunks in range - for dim_chunk_ix in range(dim_chunk_ix_from, dim_chunk_ix_to): - # compute offsets for chunk within overall array - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_limit = min(self.dim_len, (dim_chunk_ix + 1) * self.dim_chunk_len) - - # determine chunk length, accounting for trailing chunk - dim_chunk_len = dim_limit - dim_offset - - if self.start < dim_offset: - # selection starts before current chunk - dim_chunk_sel_start = 0 - remainder = (dim_offset - self.start) % self.step - if remainder: - dim_chunk_sel_start += self.step - remainder - # compute number of previous items, provides offset into output array - dim_out_offset = ceildiv((dim_offset - self.start), self.step) - - else: - # selection starts within current chunk - dim_chunk_sel_start = self.start - dim_offset - dim_out_offset = 0 - - if self.stop > dim_limit: - # selection ends after current chunk - dim_chunk_sel_stop = dim_chunk_len - - else: - # selection ends within current chunk - dim_chunk_sel_stop = self.stop - dim_offset - - dim_chunk_sel = slice(dim_chunk_sel_start, dim_chunk_sel_stop, self.step) - dim_chunk_nitems = ceildiv((dim_chunk_sel_stop - dim_chunk_sel_start), self.step) - - # If there are no elements on the selection within this chunk, then skip - if dim_chunk_nitems == 0: - continue - - dim_out_sel = slice(dim_out_offset, dim_out_offset + dim_chunk_nitems) - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -def check_selection_length(selection, shape): - if len(selection) > len(shape): - err_too_many_indices(selection, shape) - - -def replace_ellipsis(selection, shape): - selection = ensure_tuple(selection) - - # count number of ellipsis present - n_ellipsis = sum(1 for i in selection if i is Ellipsis) - - if n_ellipsis > 1: - # more than 1 is an error - raise IndexError("an index can only have a single ellipsis ('...')") - - elif n_ellipsis == 1: - # locate the ellipsis, count how many items to left and right - n_items_l = selection.index(Ellipsis) # items to left of ellipsis - n_items_r = len(selection) - (n_items_l + 1) # items to right of ellipsis - n_items = len(selection) - 1 # all non-ellipsis items - - if n_items >= len(shape): - # ellipsis does nothing, just remove it - selection = tuple(i for i in selection if i != Ellipsis) - - else: - # replace ellipsis with as many slices are needed for number of dims - new_item = selection[:n_items_l] + ((slice(None),) * (len(shape) - n_items)) - if n_items_r: - new_item += selection[-n_items_r:] - selection = new_item - - # fill out selection if not completely specified - if len(selection) < len(shape): - selection += (slice(None),) * (len(shape) - len(selection)) - - # check selection not too long - check_selection_length(selection, shape) - - return selection - - -def replace_lists(selection): - return tuple( - np.asarray(dim_sel) if isinstance(dim_sel, list) else dim_sel for dim_sel in selection - ) - - -def ensure_tuple(v): - if not isinstance(v, tuple): - v = (v,) - return v - - -ChunkProjection = collections.namedtuple( - "ChunkProjection", ("chunk_coords", "chunk_selection", "out_selection") -) -"""A mapping of items from chunk to output array. Can be used to extract items from the -chunk array for loading into an output array. Can also be used to extract items from a -value array for setting/updating in a chunk array. - -Parameters ----------- -chunk_coords - Indices of chunk. -chunk_selection - Selection of items from chunk array. -out_selection - Selection of items in target (output) array. - -""" - - -def is_slice(s): - return isinstance(s, slice) - - -def is_contiguous_slice(s): - return is_slice(s) and (s.step is None or s.step == 1) - - -def is_positive_slice(s): - return is_slice(s) and (s.step is None or s.step >= 1) - - -def is_contiguous_selection(selection): - selection = ensure_tuple(selection) - return all((is_integer_array(s) or is_contiguous_slice(s) or s == Ellipsis) for s in selection) - - -def is_basic_selection(selection): - selection = ensure_tuple(selection) - return all(is_integer(s) or is_positive_slice(s) for s in selection) - - -# noinspection PyProtectedMember -class BasicIndexer: - def __init__(self, selection, array): - # handle ellipsis - selection = replace_ellipsis(selection, array._shape) - - # setup per-dimension indexers - dim_indexers = [] - for dim_sel, dim_len, dim_chunk_len in zip(selection, array._shape, array._chunks): - if is_integer(dim_sel): - dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif is_slice(dim_sel): - dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_chunk_len) - - else: - raise IndexError( - "unsupported selection item for basic indexing; " - "expected integer or slice, got {!r}".format(type(dim_sel)) - ) - - dim_indexers.append(dim_indexer) - - self.dim_indexers = dim_indexers - self.shape = tuple(s.nitems for s in self.dim_indexers if not isinstance(s, IntDimIndexer)) - self.drop_axes = () - - def __iter__(self): - for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) - chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) - out_selection = tuple( - p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None - ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) - - -class BoolArrayDimIndexer: - def __init__(self, dim_sel, dim_len, dim_chunk_len): - # check number of dimensions - if not is_bool_array(dim_sel, 1): - raise IndexError("Boolean arrays in an orthogonal selection must be 1-dimensional only") - - # check shape - if dim_sel.shape[0] != dim_len: - raise IndexError( - "Boolean array has the wrong length for dimension; expected {}, got {}".format( - dim_len, dim_sel.shape[0] - ) - ) - - # store attributes - self.dim_sel = dim_sel - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) - - # precompute number of selected items for each chunk - self.chunk_nitems = np.zeros(self.nchunks, dtype="i8") - for dim_chunk_ix in range(self.nchunks): - dim_offset = dim_chunk_ix * self.dim_chunk_len - self.chunk_nitems[dim_chunk_ix] = np.count_nonzero( - self.dim_sel[dim_offset : dim_offset + self.dim_chunk_len] - ) - self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) - self.nitems = self.chunk_nitems_cumsum[-1] - self.dim_chunk_ixs = np.nonzero(self.chunk_nitems)[0] - - def __iter__(self): - # iterate over chunks with at least one item - for dim_chunk_ix in self.dim_chunk_ixs: - # find region in chunk - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_chunk_sel = self.dim_sel[dim_offset : dim_offset + self.dim_chunk_len] - - # pad out if final chunk - if dim_chunk_sel.shape[0] < self.dim_chunk_len: - tmp = np.zeros(self.dim_chunk_len, dtype=bool) - tmp[: dim_chunk_sel.shape[0]] = dim_chunk_sel - dim_chunk_sel = tmp - - # find region in output - if dim_chunk_ix == 0: - start = 0 - else: - start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] - stop = self.chunk_nitems_cumsum[dim_chunk_ix] - dim_out_sel = slice(start, stop) - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -class Order: - UNKNOWN = 0 - INCREASING = 1 - DECREASING = 2 - UNORDERED = 3 - - @staticmethod - def check(a): - diff = np.diff(a) - diff_positive = diff >= 0 - n_diff_positive = np.count_nonzero(diff_positive) - all_increasing = n_diff_positive == len(diff_positive) - any_increasing = n_diff_positive > 0 - if all_increasing: - order = Order.INCREASING - elif any_increasing: - order = Order.UNORDERED - else: - order = Order.DECREASING - return order - - -def wraparound_indices(x, dim_len): - loc_neg = x < 0 - if np.any(loc_neg): - x[loc_neg] = x[loc_neg] + dim_len - - -def boundscheck_indices(x, dim_len): - if np.any(x < 0) or np.any(x >= dim_len): - raise BoundsCheckError(dim_len) - - -class IntArrayDimIndexer: - """Integer array selection against a single dimension.""" - - def __init__( - self, - dim_sel, - dim_len, - dim_chunk_len, - wraparound=True, - boundscheck=True, - order=Order.UNKNOWN, - ): - # ensure 1d array - dim_sel = np.asanyarray(dim_sel) - if not is_integer_array(dim_sel, 1): - raise IndexError("integer arrays in an orthogonal selection must be 1-dimensional only") - - # handle wraparound - if wraparound: - wraparound_indices(dim_sel, dim_len) - - # handle out of bounds - if boundscheck: - boundscheck_indices(dim_sel, dim_len) - - # store attributes - self.dim_len = dim_len - self.dim_chunk_len = dim_chunk_len - self.nchunks = ceildiv(self.dim_len, self.dim_chunk_len) - self.nitems = len(dim_sel) - - # determine which chunk is needed for each selection item - # note: for dense integer selections, the division operation here is the - # bottleneck - dim_sel_chunk = dim_sel // dim_chunk_len - - # determine order of indices - if order == Order.UNKNOWN: - order = Order.check(dim_sel) - self.order = order - - if self.order == Order.INCREASING: - self.dim_sel = dim_sel - self.dim_out_sel = None - elif self.order == Order.DECREASING: - self.dim_sel = dim_sel[::-1] - # TODO should be possible to do this without creating an arange - self.dim_out_sel = np.arange(self.nitems - 1, -1, -1) - else: - # sort indices to group by chunk - self.dim_out_sel = np.argsort(dim_sel_chunk) - self.dim_sel = np.take(dim_sel, self.dim_out_sel) - - # precompute number of selected items for each chunk - self.chunk_nitems = np.bincount(dim_sel_chunk, minlength=self.nchunks) - - # find chunks that we need to visit - self.dim_chunk_ixs = np.nonzero(self.chunk_nitems)[0] - - # compute offsets into the output array - self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) - - def __iter__(self): - for dim_chunk_ix in self.dim_chunk_ixs: - # find region in output - if dim_chunk_ix == 0: - start = 0 - else: - start = self.chunk_nitems_cumsum[dim_chunk_ix - 1] - stop = self.chunk_nitems_cumsum[dim_chunk_ix] - if self.order == Order.INCREASING: - dim_out_sel = slice(start, stop) - else: - dim_out_sel = self.dim_out_sel[start:stop] - - # find region in chunk - dim_offset = dim_chunk_ix * self.dim_chunk_len - dim_chunk_sel = self.dim_sel[start:stop] - dim_offset - - yield ChunkDimProjection(dim_chunk_ix, dim_chunk_sel, dim_out_sel) - - -def slice_to_range(s: slice, l: int): # noqa: E741 - return range(*s.indices(l)) - - -def ix_(selection, shape): - """Convert an orthogonal selection to a numpy advanced (fancy) selection, like ``numpy.ix_`` - but with support for slices and single ints.""" - - # normalisation - selection = replace_ellipsis(selection, shape) - - # replace slice and int as these are not supported by numpy.ix_ - selection = [ - slice_to_range(dim_sel, dim_len) - if isinstance(dim_sel, slice) - else [dim_sel] - if is_integer(dim_sel) - else dim_sel - for dim_sel, dim_len in zip(selection, shape) - ] - - # now get numpy to convert to a coordinate selection - selection = np.ix_(*selection) - - return selection - - -def oindex(a, selection): - """Implementation of orthogonal indexing with slices and ints.""" - selection = replace_ellipsis(selection, a.shape) - drop_axes = tuple(i for i, s in enumerate(selection) if is_integer(s)) - selection = ix_(selection, a.shape) - result = a[selection] - if drop_axes: - result = result.squeeze(axis=drop_axes) - return result - - -def oindex_set(a, selection, value): - selection = replace_ellipsis(selection, a.shape) - drop_axes = tuple(i for i, s in enumerate(selection) if is_integer(s)) - selection = ix_(selection, a.shape) - if not np.isscalar(value) and drop_axes: - value = np.asanyarray(value) - value_selection = [slice(None)] * len(a.shape) - for i in drop_axes: - value_selection[i] = np.newaxis - value_selection = tuple(value_selection) - value = value[value_selection] - a[selection] = value - - -# noinspection PyProtectedMember -class OrthogonalIndexer: - def __init__(self, selection, array): - # handle ellipsis - selection = replace_ellipsis(selection, array._shape) - - # normalize list to array - selection = replace_lists(selection) - - # setup per-dimension indexers - dim_indexers = [] - for dim_sel, dim_len, dim_chunk_len in zip(selection, array._shape, array._chunks): - if is_integer(dim_sel): - dim_indexer = IntDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif isinstance(dim_sel, slice): - dim_indexer = SliceDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif is_integer_array(dim_sel): - dim_indexer = IntArrayDimIndexer(dim_sel, dim_len, dim_chunk_len) - - elif is_bool_array(dim_sel): - dim_indexer = BoolArrayDimIndexer(dim_sel, dim_len, dim_chunk_len) - - else: - raise IndexError( - "unsupported selection item for orthogonal indexing; " - "expected integer, slice, integer array or Boolean " - "array, got {!r}".format(type(dim_sel)) - ) - - dim_indexers.append(dim_indexer) - - self.array = array - self.dim_indexers = dim_indexers - self.shape = tuple(s.nitems for s in self.dim_indexers if not isinstance(s, IntDimIndexer)) - self.is_advanced = not is_basic_selection(selection) - if self.is_advanced: - self.drop_axes = tuple( - i - for i, dim_indexer in enumerate(self.dim_indexers) - if isinstance(dim_indexer, IntDimIndexer) - ) - else: - self.drop_axes = () - - def __iter__(self): - for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) - chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) - out_selection = tuple( - p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None - ) - - # handle advanced indexing arrays orthogonally - if self.is_advanced: - # N.B., numpy doesn't support orthogonal indexing directly as yet, - # so need to work around via np.ix_. Also np.ix_ does not support a - # mixture of arrays and slices or integers, so need to convert slices - # and integers into ranges. - chunk_selection = ix_(chunk_selection, self.array._chunks) - - # special case for non-monotonic indices - if not is_basic_selection(out_selection): - out_selection = ix_(out_selection, self.shape) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) - - -class OIndex: - def __init__(self, array): - self.array = array - - def __getitem__(self, selection): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - return self.array.get_orthogonal_selection(selection, fields=fields) - - def __setitem__(self, selection, value): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - return self.array.set_orthogonal_selection(selection, value, fields=fields) - - -# noinspection PyProtectedMember -class BlockIndexer: - def __init__(self, selection, array): - # handle ellipsis - selection = replace_ellipsis(selection, array._shape) - - # normalize list to array - selection = replace_lists(selection) - - # setup per-dimension indexers - dim_indexers = [] - for dim_sel, dim_len, dim_chunk_size in zip(selection, array._shape, array._chunks): - dim_numchunks = int(np.ceil(dim_len / dim_chunk_size)) - - if is_integer(dim_sel): - if dim_sel < 0: - dim_sel = dim_numchunks + dim_sel - - start = dim_sel * dim_chunk_size - stop = start + dim_chunk_size - slice_ = slice(start, stop) - - elif is_slice(dim_sel): - start = dim_sel.start if dim_sel.start is not None else 0 - stop = dim_sel.stop if dim_sel.stop is not None else dim_numchunks - - if dim_sel.step not in {1, None}: - raise IndexError( - "unsupported selection item for block indexing; " - "expected integer or slice with step=1, got {!r}".format(type(dim_sel)) - ) - - # Can't reuse wraparound_indices because it expects a numpy array - # We have integers here. - if start < 0: - start = dim_numchunks + start - if stop < 0: - stop = dim_numchunks + stop - - start = start * dim_chunk_size - stop = stop * dim_chunk_size - slice_ = slice(start, stop) - - else: - raise IndexError( - "unsupported selection item for block indexing; " - "expected integer or slice, got {!r}".format(type(dim_sel)) - ) - - dim_indexer = SliceDimIndexer(slice_, dim_len, dim_chunk_size) - dim_indexers.append(dim_indexer) - - if start >= dim_len or start < 0: - raise BoundsCheckError(dim_len) - - self.dim_indexers = dim_indexers - self.shape = tuple(s.nitems for s in self.dim_indexers) - self.drop_axes = () - - def __iter__(self): - for dim_projections in itertools.product(*self.dim_indexers): - chunk_coords = tuple(p.dim_chunk_ix for p in dim_projections) - chunk_selection = tuple(p.dim_chunk_sel for p in dim_projections) - out_selection = tuple( - p.dim_out_sel for p in dim_projections if p.dim_out_sel is not None - ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) - - -class BlockIndex: - def __init__(self, array): - self.array = array - - def __getitem__(self, selection): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - return self.array.get_block_selection(selection, fields=fields) - - def __setitem__(self, selection, value): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - return self.array.set_block_selection(selection, value, fields=fields) - - -# noinspection PyProtectedMember -def is_coordinate_selection(selection, array): - return (len(selection) == len(array._shape)) and all( - is_integer(dim_sel) or is_integer_array(dim_sel) for dim_sel in selection - ) - - -# noinspection PyProtectedMember -def is_mask_selection(selection, array): - return ( - len(selection) == 1 and is_bool_array(selection[0]) and selection[0].shape == array._shape - ) - - -# noinspection PyProtectedMember -class CoordinateIndexer: - def __init__(self, selection, array): - # some initial normalization - selection = ensure_tuple(selection) - selection = tuple([i] if is_integer(i) else i for i in selection) - selection = replace_lists(selection) - - # validation - if not is_coordinate_selection(selection, array): - raise IndexError( - "invalid coordinate selection; expected one integer " - "(coordinate) array per dimension of the target array, " - "got {!r}".format(selection) - ) - - # handle wraparound, boundscheck - for dim_sel, dim_len in zip(selection, array.shape): - # handle wraparound - wraparound_indices(dim_sel, dim_len) - - # handle out of bounds - boundscheck_indices(dim_sel, dim_len) - - # compute chunk index for each point in the selection - chunks_multi_index = tuple( - dim_sel // dim_chunk_len for (dim_sel, dim_chunk_len) in zip(selection, array._chunks) - ) - - # broadcast selection - this will raise error if array dimensions don't match - selection = np.broadcast_arrays(*selection) - chunks_multi_index = np.broadcast_arrays(*chunks_multi_index) - - # remember shape of selection, because we will flatten indices for processing - self.sel_shape = selection[0].shape if selection[0].shape else (1,) - - # flatten selection - selection = [dim_sel.reshape(-1) for dim_sel in selection] - chunks_multi_index = [dim_chunks.reshape(-1) for dim_chunks in chunks_multi_index] - - # ravel chunk indices - chunks_raveled_indices = np.ravel_multi_index(chunks_multi_index, dims=array._cdata_shape) - - # group points by chunk - if np.any(np.diff(chunks_raveled_indices) < 0): - # optimisation, only sort if needed - sel_sort = np.argsort(chunks_raveled_indices) - selection = tuple(dim_sel[sel_sort] for dim_sel in selection) - else: - sel_sort = None - - # store attributes - self.selection = selection - self.sel_sort = sel_sort - self.shape = selection[0].shape if selection[0].shape else (1,) - self.drop_axes = () - self.array = array - - # precompute number of selected items for each chunk - self.chunk_nitems = np.bincount(chunks_raveled_indices, minlength=array.nchunks) - self.chunk_nitems_cumsum = np.cumsum(self.chunk_nitems) - # locate the chunks we need to process - self.chunk_rixs = np.nonzero(self.chunk_nitems)[0] - - # unravel chunk indices - self.chunk_mixs = np.unravel_index(self.chunk_rixs, array._cdata_shape) - - def __iter__(self): - # iterate over chunks - for i, chunk_rix in enumerate(self.chunk_rixs): - chunk_coords = tuple(m[i] for m in self.chunk_mixs) - if chunk_rix == 0: - start = 0 - else: - start = self.chunk_nitems_cumsum[chunk_rix - 1] - stop = self.chunk_nitems_cumsum[chunk_rix] - if self.sel_sort is None: - out_selection = slice(start, stop) - else: - out_selection = self.sel_sort[start:stop] - - chunk_offsets = tuple( - dim_chunk_ix * dim_chunk_len - for dim_chunk_ix, dim_chunk_len in zip(chunk_coords, self.array._chunks) - ) - chunk_selection = tuple( - dim_sel[start:stop] - dim_chunk_offset - for (dim_sel, dim_chunk_offset) in zip(self.selection, chunk_offsets) - ) - - yield ChunkProjection(chunk_coords, chunk_selection, out_selection) - - -# noinspection PyProtectedMember -class MaskIndexer(CoordinateIndexer): - def __init__(self, selection, array): - # some initial normalization - selection = ensure_tuple(selection) - selection = replace_lists(selection) - - # validation - if not is_mask_selection(selection, array): - raise IndexError( - "invalid mask selection; expected one Boolean (mask)" - "array with the same shape as the target array, got {!r}".format(selection) - ) - - # convert to indices - selection = np.nonzero(selection[0]) - - # delegate the rest to superclass - super().__init__(selection, array) - - -class VIndex: - def __init__(self, array): - self.array = array - - def __getitem__(self, selection): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - if is_coordinate_selection(selection, self.array): - return self.array.get_coordinate_selection(selection, fields=fields) - elif is_mask_selection(selection, self.array): - return self.array.get_mask_selection(selection, fields=fields) - else: - raise VindexInvalidSelectionError(selection) - - def __setitem__(self, selection, value): - fields, selection = pop_fields(selection) - selection = ensure_tuple(selection) - selection = replace_lists(selection) - if is_coordinate_selection(selection, self.array): - self.array.set_coordinate_selection(selection, value, fields=fields) - elif is_mask_selection(selection, self.array): - self.array.set_mask_selection(selection, value, fields=fields) - else: - raise VindexInvalidSelectionError(selection) - - -def check_fields(fields, dtype): - # early out - if fields is None: - return dtype - # check type - if not isinstance(fields, (str, list, tuple)): - raise IndexError( - "'fields' argument must be a string or list of strings; found {!r}".format(type(fields)) - ) - if fields: - if dtype.names is None: - raise IndexError("invalid 'fields' argument, array does not have any fields") - try: - if isinstance(fields, str): - # single field selection - out_dtype = dtype[fields] - else: - # multiple field selection - out_dtype = np.dtype([(f, dtype[f]) for f in fields]) - except KeyError as e: - raise IndexError("invalid 'fields' argument, field not found: {!r}".format(e)) - else: - return out_dtype - else: - return dtype - - -def check_no_multi_fields(fields): - if isinstance(fields, list): - if len(fields) == 1: - return fields[0] - elif len(fields) > 1: - raise IndexError("multiple fields are not supported for this operation") - return fields - - -def pop_fields(selection): - if isinstance(selection, str): - # single field selection - fields = selection - selection = () - elif not isinstance(selection, tuple): - # single selection item, no fields - fields = None - # leave selection as-is - else: - # multiple items, split fields from selection items - fields = [f for f in selection if isinstance(f, str)] - fields = fields[0] if len(fields) == 1 else fields - selection = tuple(s for s in selection if not isinstance(s, str)) - selection = selection[0] if len(selection) == 1 else selection - return fields, selection - - -def make_slice_selection(selection): - ls = [] - for dim_selection in selection: - if is_integer(dim_selection): - ls.append(slice(int(dim_selection), int(dim_selection) + 1, 1)) - elif isinstance(dim_selection, np.ndarray): - if len(dim_selection) == 1: - ls.append(slice(int(dim_selection[0]), int(dim_selection[0]) + 1, 1)) - else: - raise ArrayIndexError - else: - ls.append(dim_selection) - return ls - - -class PartialChunkIterator: - """Iterator to retrieve the specific coordinates of requested data - from within a compressed chunk. - - Parameters - ---------- - selection : tuple - tuple of slice objects to take from the chunk - arr_shape : shape of chunk to select data from - - Attributes - ---------- - arr_shape - selection - - Returns - ------- - Tuple with 3 elements: - - start: int - elements offset in the chunk to read from - nitems: int - number of elements to read in the chunk from start - partial_out_selection: list of slices - indices of a temporary empty array of size `Array._chunks` to assign - the decompressed data to after the partial read. - - Notes - ----- - An array is flattened when compressed with blosc, so this iterator takes - the wanted selection of an array and determines the wanted coordinates - of the flattened, compressed data to be read and then decompressed. The - decompressed data is then placed in a temporary empty array of size - `Array._chunks` at the indices yielded as partial_out_selection. - Once all the slices yielded by this iterator have been read, decompressed - and written to the temporary array, the wanted slice of the chunk can be - indexed from the temporary array and written to the out_selection slice - of the out array. - - """ - - def __init__(self, selection, arr_shape): - selection = make_slice_selection(selection) - self.arr_shape = arr_shape - - # number of selection dimensions can't be greater than the number of chunk dimensions - if len(selection) > len(self.arr_shape): - raise ValueError( - "Selection has more dimensions then the array:\n" - f"selection dimensions = {len(selection)}\n" - f"array dimensions = {len(self.arr_shape)}" - ) - - # any selection can not be out of the range of the chunk - selection_shape = np.empty(self.arr_shape)[tuple(selection)].shape - if any( - selection_dim < 0 or selection_dim > arr_dim - for selection_dim, arr_dim in zip(selection_shape, self.arr_shape) - ): - raise IndexError( - "a selection index is out of range for the dimension" - ) # pragma: no cover - - for i, dim_size in enumerate(self.arr_shape[::-1]): - index = len(self.arr_shape) - (i + 1) - if index <= len(selection) - 1: - slice_size = selection_shape[index] - if slice_size == dim_size and index > 0: - selection.pop() - else: - break - - chunk_loc_slices = [] - last_dim_slice = None if selection[-1].step > 1 else selection.pop() - for arr_shape_i, sl in zip(arr_shape, selection): - dim_chunk_loc_slices = [] - assert isinstance(sl, slice) - for x in slice_to_range(sl, arr_shape_i): - dim_chunk_loc_slices.append(slice(x, x + 1, 1)) - chunk_loc_slices.append(dim_chunk_loc_slices) - if last_dim_slice: - chunk_loc_slices.append([last_dim_slice]) - self.chunk_loc_slices = list(itertools.product(*chunk_loc_slices)) - - def __iter__(self): - chunk1 = self.chunk_loc_slices[0] - nitems = (chunk1[-1].stop - chunk1[-1].start) * np.prod( - self.arr_shape[len(chunk1) :], dtype=int - ) - for partial_out_selection in self.chunk_loc_slices: - start = 0 - for i, sl in enumerate(partial_out_selection): - start += sl.start * np.prod(self.arr_shape[i + 1 :], dtype=int) - yield start, nitems, partial_out_selection diff --git a/src/zarr/v2/meta.py b/src/zarr/v2/meta.py deleted file mode 100644 index 2f7ce1242e..0000000000 --- a/src/zarr/v2/meta.py +++ /dev/null @@ -1,302 +0,0 @@ -import base64 -import itertools -from collections.abc import Mapping - -import numpy as np - -from zarr.v2.errors import MetadataError -from zarr.v2.util import json_dumps, json_loads - -from typing import cast, Union, Any, List, Mapping as MappingType, TYPE_CHECKING - -if TYPE_CHECKING: # pragma: no cover - pass - - -ZARR_FORMAT = 2 - -# FLOAT_FILLS = {"NaN": np.nan, "Infinity": np.PINF, "-Infinity": np.NINF} - -_v3_core_types = {"bool", "i1", "u1"} | set("".join(d) for d in itertools.product("<>", ("u", "i", "f"), ("2", "4", "8"))) - -# The set of complex types allowed ({"c8", ">c16"}) -_v3_complex_types = set(f"{end}c{_bytes}" for end, _bytes in itertools.product("<>", ("8", "16"))) - -# All dtype.str values corresponding to datetime64 and timedelta64 -# see: https://numpy.org/doc/stable/reference/arrays.datetime.html#datetime-units -_date_units = ["Y", "M", "W", "D"] -_time_units = ["h", "m", "s", "ms", "us", "μs", "ns", "ps", "fs", "as"] -_v3_datetime_types = set( - f"{end}{kind}8[{unit}]" - for end, unit, kind in itertools.product("<>", _date_units + _time_units, ("m", "M")) -) - - -def get_extended_dtype_info(dtype) -> dict: - if dtype.str in _v3_complex_types: - return dict( - extension="https://zarr-specs.readthedocs.io/en/core-protocol-v3.0-dev/protocol/extensions/complex-dtypes/v1.0.html", - type=dtype.str, - fallback=None, - ) - elif dtype.str == "|O": - return dict( - extension="TODO: object array protocol URL", - type=dtype.str, - fallback=None, - ) - elif dtype.str.startswith("|S"): - return dict( - extension="TODO: bytestring array protocol URL", - type=dtype.str, - fallback=None, - ) - elif dtype.str.startswith("U"): - return dict( - extension="TODO: unicode array protocol URL", - type=dtype.str, - fallback=None, - ) - elif dtype.str.startswith("|V"): - return dict( - extension="TODO: structured array protocol URL", - type=dtype.descr, - fallback=None, - ) - elif dtype.str in _v3_datetime_types: - return dict( - extension="https://zarr-specs.readthedocs.io/en/latest/extensions/data-types/datetime/v1.0.html", - type=dtype.str, - fallback=None, - ) - else: - raise ValueError(f"Unsupported dtype: {dtype}") - - -class Metadata2: - ZARR_FORMAT = ZARR_FORMAT - - @classmethod - def parse_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]: - # Here we allow that a store may return an already-parsed metadata object, - # or a string of JSON that we will parse here. We allow for an already-parsed - # object to accommodate a consolidated metadata store, where all the metadata for - # all groups and arrays will already have been parsed from JSON. - - if isinstance(s, Mapping): - # assume metadata has already been parsed into a mapping object - meta = s - - else: - # assume metadata needs to be parsed as JSON - meta = json_loads(s) - - return meta - - @classmethod - def decode_array_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]: - meta = cls.parse_metadata(s) - - # check metadata format - zarr_format = meta.get("zarr_format", None) - if zarr_format != cls.ZARR_FORMAT: - raise MetadataError("unsupported zarr format: %s" % zarr_format) - - # extract array metadata fields - try: - dtype = cls.decode_dtype(meta["dtype"]) - if dtype.hasobject: - import numcodecs - - object_codec = numcodecs.get_codec(meta["filters"][0]) - else: - object_codec = None - - dimension_separator = meta.get("dimension_separator", None) - fill_value = cls.decode_fill_value(meta["fill_value"], dtype, object_codec) - meta = dict( - zarr_format=meta["zarr_format"], - shape=tuple(meta["shape"]), - chunks=tuple(meta["chunks"]), - dtype=dtype, - compressor=meta["compressor"], - fill_value=fill_value, - order=meta["order"], - filters=meta["filters"], - ) - if dimension_separator: - meta["dimension_separator"] = dimension_separator - except Exception as e: - raise MetadataError("error decoding metadata") from e - else: - return meta - - @classmethod - def encode_array_metadata(cls, meta: MappingType[str, Any]) -> bytes: - dtype = meta["dtype"] - sdshape = () - if dtype.subdtype is not None: - dtype, sdshape = dtype.subdtype - - dimension_separator = meta.get("dimension_separator") - if dtype.hasobject: - import numcodecs - - object_codec = numcodecs.get_codec(meta["filters"][0]) - else: - object_codec = None - - meta = dict( - zarr_format=cls.ZARR_FORMAT, - shape=meta["shape"] + sdshape, - chunks=meta["chunks"], - dtype=cls.encode_dtype(dtype), - compressor=meta["compressor"], - fill_value=cls.encode_fill_value(meta["fill_value"], dtype, object_codec), - order=meta["order"], - filters=meta["filters"], - ) - if dimension_separator: - meta["dimension_separator"] = dimension_separator - - return json_dumps(meta) - - @classmethod - def encode_dtype(cls, d: np.dtype): - if d.fields is None: - return d.str - else: - return d.descr - - @classmethod - def _decode_dtype_descr(cls, d) -> List[Any]: - # need to convert list of lists to list of tuples - if isinstance(d, list): - # recurse to handle nested structures - d = [(k[0], cls._decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d] - return d - - @classmethod - def decode_dtype(cls, d) -> np.dtype: - d = cls._decode_dtype_descr(d) - return np.dtype(d) - - @classmethod - def decode_group_metadata(cls, s: Union[MappingType, bytes, str]) -> MappingType[str, Any]: - meta = cls.parse_metadata(s) - - # check metadata format version - zarr_format = meta.get("zarr_format", None) - if zarr_format != cls.ZARR_FORMAT: - raise MetadataError("unsupported zarr format: %s" % zarr_format) - - meta = dict(zarr_format=zarr_format) - return meta - - # N.B., keep `meta` parameter as a placeholder for future - # noinspection PyUnusedLocal - @classmethod - def encode_group_metadata(cls, meta=None) -> bytes: - meta = dict(zarr_format=cls.ZARR_FORMAT) - return json_dumps(meta) - - @classmethod - def decode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> Any: - # early out - if v is None: - return v - if dtype.kind == "V" and dtype.hasobject: - if object_codec is None: - raise ValueError("missing object_codec for object array") - v = base64.standard_b64decode(v) - v = object_codec.decode(v) - v = np.array(v, dtype=dtype)[()] - return v - if dtype.kind == "f": - if v == "NaN": - return np.nan - elif v == "Infinity": - return np.inf - elif v == "-Infinity": - return -np.inf - else: - return np.array(v, dtype=dtype)[()] - elif dtype.kind == "c": - v = ( - cls.decode_fill_value(v[0], dtype.type().real.dtype), - cls.decode_fill_value(v[1], dtype.type().imag.dtype), - ) - v = v[0] + 1j * v[1] - return np.array(v, dtype=dtype)[()] - elif dtype.kind == "S": - # noinspection PyBroadException - try: - v = base64.standard_b64decode(v) - except Exception: - # be lenient, allow for other values that may have been used before base64 - # encoding and may work as fill values, e.g., the number 0 - pass - v = np.array(v, dtype=dtype)[()] - return v - elif dtype.kind == "V": - v = base64.standard_b64decode(v) - v = np.array(v, dtype=dtype.str).view(dtype)[()] - return v - elif dtype.kind == "U": - # leave as-is - return v - else: - return np.array(v, dtype=dtype)[()] - - @classmethod - def encode_fill_value(cls, v: Any, dtype: np.dtype, object_codec: Any = None) -> Any: - # early out - if v is None: - return v - if dtype.kind == "V" and dtype.hasobject: - if object_codec is None: - raise ValueError("missing object_codec for object array") - v = object_codec.encode(v) - v = str(base64.standard_b64encode(v), "ascii") - return v - if dtype.kind == "f": - if np.isnan(v): - return "NaN" - elif np.isposinf(v): - return "Infinity" - elif np.isneginf(v): - return "-Infinity" - else: - return float(v) - elif dtype.kind in ("u", "i"): - return int(v) - elif dtype.kind == "b": - return bool(v) - elif dtype.kind == "c": - c = cast(np.complex128, np.dtype(complex).type()) - v = ( - cls.encode_fill_value(v.real, c.real.dtype, object_codec), - cls.encode_fill_value(v.imag, c.imag.dtype, object_codec), - ) - return v - elif dtype.kind in ("S", "V"): - v = str(base64.standard_b64encode(v), "ascii") - return v - elif dtype.kind == "U": - return v - elif dtype.kind in ("m", "M"): - return int(v.view("i8")) - else: - return v - - -parse_metadata = Metadata2.parse_metadata -decode_array_metadata = Metadata2.decode_array_metadata -encode_array_metadata = Metadata2.encode_array_metadata -encode_dtype = Metadata2.encode_dtype -_decode_dtype_descr = Metadata2._decode_dtype_descr -decode_dtype = Metadata2.decode_dtype -decode_group_metadata = Metadata2.decode_group_metadata -encode_group_metadata = Metadata2.encode_group_metadata -decode_fill_value = Metadata2.decode_fill_value -encode_fill_value = Metadata2.encode_fill_value diff --git a/src/zarr/v2/meta_v1.py b/src/zarr/v2/meta_v1.py deleted file mode 100644 index 881b9191eb..0000000000 --- a/src/zarr/v2/meta_v1.py +++ /dev/null @@ -1,64 +0,0 @@ -import json - -import numpy as np - -from zarr.v2.errors import MetadataError - - -def decode_metadata(b): - s = str(b, "ascii") - meta = json.loads(s) - zarr_format = meta.get("zarr_format", None) - if zarr_format != 1: - raise MetadataError("unsupported zarr format: %s" % zarr_format) - try: - meta = dict( - zarr_format=meta["zarr_format"], - shape=tuple(meta["shape"]), - chunks=tuple(meta["chunks"]), - dtype=decode_dtype(meta["dtype"]), - compression=meta["compression"], - compression_opts=meta["compression_opts"], - fill_value=meta["fill_value"], - order=meta["order"], - ) - except Exception as e: - raise MetadataError("error decoding metadata: %s" % e) - else: - return meta - - -def encode_metadata(meta): - meta = dict( - zarr_format=1, - shape=meta["shape"], - chunks=meta["chunks"], - dtype=encode_dtype(meta["dtype"]), - compression=meta["compression"], - compression_opts=meta["compression_opts"], - fill_value=meta["fill_value"], - order=meta["order"], - ) - s = json.dumps(meta, indent=4, sort_keys=True, ensure_ascii=True) - b = s.encode("ascii") - return b - - -def encode_dtype(d): - if d.fields is None: - return d.str - else: - return d.descr - - -def _decode_dtype_descr(d): - # need to convert list of lists to list of tuples - if isinstance(d, list): - # recurse to handle nested structures - d = [(f, _decode_dtype_descr(v)) for f, v in d] - return d - - -def decode_dtype(d): - d = _decode_dtype_descr(d) - return np.dtype(d) diff --git a/src/zarr/v2/n5.py b/src/zarr/v2/n5.py deleted file mode 100644 index ece110f49d..0000000000 --- a/src/zarr/v2/n5.py +++ /dev/null @@ -1,897 +0,0 @@ -"""This module contains a storage class and codec to support the N5 format.""" - -import os -import struct -import sys -from typing import Any, Dict, Optional, cast -import warnings - -import numpy as np -from numcodecs.abc import Codec -from numcodecs.compat import ndarray_copy -from numcodecs.registry import get_codec, register_codec - -from zarr.v2.meta import ZARR_FORMAT, json_dumps, json_loads -from zarr.v2.storage import FSStore -from zarr.v2.storage import NestedDirectoryStore, _prog_ckey, _prog_number, normalize_storage_path -from zarr.v2.storage import array_meta_key as zarr_array_meta_key -from zarr.v2.storage import attrs_key as zarr_attrs_key -from zarr.v2.storage import group_meta_key as zarr_group_meta_key - -N5_FORMAT = "2.0.0" - -zarr_to_n5_keys = [ - ("chunks", "blockSize"), - ("dtype", "dataType"), - ("compressor", "compression"), - ("shape", "dimensions"), -] -n5_attrs_key = "attributes.json" -n5_keywords = ["n5", "dataType", "dimensions", "blockSize", "compression"] - - -class N5Store(NestedDirectoryStore): - """Storage class using directories and files on a standard file system, - following the N5 format (https://github.com/saalfeldlab/n5). - - Parameters - ---------- - path : string - Location of directory to use as the root of the storage hierarchy. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-sensitive and - case-insensitive file system. Default value is False. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.N5Store('data/array.n5') - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Store a group:: - - >>> store = zarr.N5Store('data/group.n5') - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - - Notes - ----- - - This is an experimental feature. - - Safe to write in multiple threads or processes. - - """ - - def __getitem__(self, key: str) -> bytes: - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) - value = group_metadata_to_zarr(self._load_n5_attrs(key_new)) - - return json_dumps(value) - - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) - top_level = key == zarr_array_meta_key - value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level) - return json_dumps(value) - - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) - value = attrs_to_zarr(self._load_n5_attrs(key_new)) - - if len(value) == 0: - raise KeyError(key_new) - else: - return json_dumps(value) - - elif is_chunk_key(key): - key_new = invert_chunk_coords(key) - - else: - key_new = key - - return super().__getitem__(key_new) - - def __setitem__(self, key: str, value: Any): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) - - n5_attrs = self._load_n5_attrs(key_new) - n5_attrs.update(**group_metadata_to_n5(json_loads(value))) - - value = json_dumps(n5_attrs) - - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) - top_level = key == zarr_array_meta_key - n5_attrs = self._load_n5_attrs(key_new) - n5_attrs.update(**array_metadata_to_n5(json_loads(value), top_level=top_level)) - value = json_dumps(n5_attrs) - - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) - - n5_attrs = self._load_n5_attrs(key_new) - zarr_attrs = json_loads(value) - - for k in n5_keywords: - if k in zarr_attrs: - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) - - # remove previous user attributes - for k in list(n5_attrs.keys()): - if k not in n5_keywords: - del n5_attrs[k] - - # add new user attributes - n5_attrs.update(**zarr_attrs) - - value = json_dumps(n5_attrs) - - elif is_chunk_key(key): - key_new = invert_chunk_coords(key) - - else: - key_new = key - - super().__setitem__(key_new, value) - - def __delitem__(self, key: str): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) - elif is_chunk_key(key): - key_new = invert_chunk_coords(key) - else: - key_new = key - - super().__delitem__(key_new) - - def __contains__(self, key): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, n5_attrs_key) - if key_new not in self: - return False - # group if not a dataset (attributes do not contain 'dimensions') - return "dimensions" not in self._load_n5_attrs(key_new) - - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, n5_attrs_key) - # array if attributes contain 'dimensions' - return "dimensions" in self._load_n5_attrs(key_new) - - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, n5_attrs_key) - return self._contains_attrs(key_new) - - elif is_chunk_key(key): - key_new = invert_chunk_coords(key) - else: - key_new = key - - return super().__contains__(key_new) - - def __eq__(self, other): - return isinstance(other, N5Store) and self.path == other.path - - def listdir(self, path: Optional[str] = None): - if path is not None: - path = invert_chunk_coords(path) - path = cast(str, path) - # We can't use NestedDirectoryStore's listdir, as it requires - # array_meta_key to be present in array directories, which this store - # doesn't provide. - children = super().listdir(path=path) - - if self._is_array(path): - # replace n5 attribute file with respective zarr attribute files - children.remove(n5_attrs_key) - children.append(zarr_array_meta_key) - if self._contains_attrs(path): - children.append(zarr_attrs_key) - - # special handling of directories containing an array to map - # inverted nested chunk keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and os.path.isdir(entry_path): - for dir_path, _, file_names in os.walk(entry_path): - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path + os.path.sep)[1] - new_child = rel_path.replace(os.path.sep, ".") - new_children.append(invert_chunk_coords(new_child)) - else: - new_children.append(entry) - - return sorted(new_children) - - elif self._is_group(path): - # replace n5 attribute file with respective zarr attribute files - children.remove(n5_attrs_key) - children.append(zarr_group_meta_key) - if self._contains_attrs(path): - children.append(zarr_attrs_key) - - return sorted(children) - - else: - return children - - def _load_n5_attrs(self, path: str) -> Dict[str, Any]: - try: - s = super().__getitem__(path) - return json_loads(s) - except KeyError: - return {} - - def _is_group(self, path: str): - if path is None: - attrs_key = n5_attrs_key - else: - attrs_key = os.path.join(path, n5_attrs_key) - - n5_attrs = self._load_n5_attrs(attrs_key) - return len(n5_attrs) > 0 and "dimensions" not in n5_attrs - - def _is_array(self, path: str): - if path is None: - attrs_key = n5_attrs_key - else: - attrs_key = os.path.join(path, n5_attrs_key) - - return "dimensions" in self._load_n5_attrs(attrs_key) - - def _contains_attrs(self, path: str): - if path is None: - attrs_key = n5_attrs_key - else: - if not path.endswith(n5_attrs_key): - attrs_key = os.path.join(path, n5_attrs_key) - else: - attrs_key = path - - attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key)) - return len(attrs) > 0 - - -class N5FSStore(FSStore): - """Implementation of the N5 format (https://github.com/saalfeldlab/n5) - using `fsspec`, which allows storage on a variety of filesystems. Based - on `zarr.N5Store`. - - Parameters - ---------- - path : string - Location of directory to use as the root of the storage hierarchy. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-sensitive and - case-insensitive file system. Default value is False. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.N5FSStore('data/array.n5', auto_mkdir=True) - >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Store a group:: - - >>> store = zarr.N5FSStore('data/group.n5', auto_mkdir=True) - >>> root = zarr.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - - Notes - ----- - This is an experimental feature. - Safe to write in multiple threads or processes. - - Be advised that the `_dimension_separator` property of this store - (and arrays it creates) is ".", but chunks saved by this store will - in fact be "/" separated, as proscribed by the N5 format. - - This is counter-intuitive (to say the least), but not arbitrary. - Chunks in N5 format are stored with reversed dimension order - relative to Zarr chunks: a chunk of a 3D Zarr array would be stored - on a file system as `/0/1/2`, but in N5 the same chunk would be - stored as `/2/1/0`. Therefore, stores targeting N5 must intercept - chunk keys and flip the order of the dimensions before writing to - storage, and this procedure requires chunk keys with "." separated - dimensions, hence the Zarr arrays targeting N5 have the deceptive - "." dimension separator. - """ - - _array_meta_key = "attributes.json" - _group_meta_key = "attributes.json" - _attrs_key = "attributes.json" - - def __init__(self, *args, **kwargs): - if "dimension_separator" in kwargs: - warnings.warn("Keyword argument `dimension_separator` will be ignored") - kwargs["dimension_separator"] = "." - super().__init__(*args, **kwargs) - - @staticmethod - def _swap_separator(key: str): - segments = list(key.split("/")) - if segments: - last_segment = segments[-1] - if _prog_ckey.match(last_segment): - coords = list(last_segment.split(".")) - last_segment = "/".join(coords[::-1]) - segments = segments[:-1] + [last_segment] - key = "/".join(segments) - return key - - def _normalize_key(self, key: str): - if is_chunk_key(key): - key = invert_chunk_coords(key) - - key = normalize_storage_path(key).lstrip("/") - if key: - *bits, end = key.split("/") - - if end not in (self._array_meta_key, self._group_meta_key, self._attrs_key): - end = end.replace(".", "/") - key = "/".join(bits + [end]) - return key.lower() if self.normalize_keys else key - - def __getitem__(self, key: str) -> bytes: - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) - value = group_metadata_to_zarr(self._load_n5_attrs(key_new)) - - return json_dumps(value) - - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) - top_level = key == zarr_array_meta_key - value = array_metadata_to_zarr(self._load_n5_attrs(key_new), top_level=top_level) - return json_dumps(value) - - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) - value = attrs_to_zarr(self._load_n5_attrs(key_new)) - - if len(value) == 0: - raise KeyError(key_new) - else: - return json_dumps(value) - - elif is_chunk_key(key): - key_new = self._swap_separator(key) - - else: - key_new = key - - return super().__getitem__(key_new) - - def __setitem__(self, key: str, value: Any): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) - - n5_attrs = self._load_n5_attrs(key_new) - n5_attrs.update(**group_metadata_to_n5(json_loads(value))) - - value = json_dumps(n5_attrs) - - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) - top_level = key == zarr_array_meta_key - n5_attrs = self._load_n5_attrs(key_new) - n5_attrs.update(**array_metadata_to_n5(json_loads(value), top_level=top_level)) - - value = json_dumps(n5_attrs) - - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) - - n5_attrs = self._load_n5_attrs(key_new) - zarr_attrs = json_loads(value) - - for k in n5_keywords: - if k in zarr_attrs.keys(): - warnings.warn(f"Attribute {k} is a reserved N5 keyword", UserWarning) - - # replace previous user attributes - for k in list(n5_attrs.keys()): - if k not in n5_keywords: - del n5_attrs[k] - - # add new user attributes - n5_attrs.update(**zarr_attrs) - - value = json_dumps(n5_attrs) - - elif is_chunk_key(key): - key_new = self._swap_separator(key) - - else: - key_new = key - - super().__setitem__(key_new, value) - - def __delitem__(self, key: str): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) - elif is_chunk_key(key): - key_new = self._swap_separator(key) - else: - key_new = key - super().__delitem__(key_new) - - def __contains__(self, key: Any): - if key.endswith(zarr_group_meta_key): - key_new = key.replace(zarr_group_meta_key, self._group_meta_key) - if key_new not in self: - return False - # group if not a dataset (attributes do not contain 'dimensions') - return "dimensions" not in self._load_n5_attrs(key_new) - - elif key.endswith(zarr_array_meta_key): - key_new = key.replace(zarr_array_meta_key, self._array_meta_key) - # array if attributes contain 'dimensions' - return "dimensions" in self._load_n5_attrs(key_new) - - elif key.endswith(zarr_attrs_key): - key_new = key.replace(zarr_attrs_key, self._attrs_key) - return self._contains_attrs(key_new) - - elif is_chunk_key(key): - key_new = self._swap_separator(key) - - else: - key_new = key - return super().__contains__(key_new) - - def __eq__(self, other: Any): - return isinstance(other, N5FSStore) and self.path == other.path - - def listdir(self, path: Optional[str] = None): - if path is not None: - path = invert_chunk_coords(path) - - # We can't use NestedDirectoryStore's listdir, as it requires - # array_meta_key to be present in array directories, which this store - # doesn't provide. - children = super().listdir(path=path) - if self._is_array(path): - # replace n5 attribute file with respective zarr attribute files - children.remove(self._array_meta_key) - children.append(zarr_array_meta_key) - if self._contains_attrs(path): - children.append(zarr_attrs_key) - - # special handling of directories containing an array to map - # inverted nested chunk keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and self.fs.isdir(entry_path): - for file_name in self.fs.find(entry_path): - file_path = os.path.join(root_path, file_name) - rel_path = file_path.split(root_path)[1] - new_child = rel_path.lstrip("/").replace("/", ".") - new_children.append(invert_chunk_coords(new_child)) - else: - new_children.append(entry) - return sorted(new_children) - - elif self._is_group(path): - # replace n5 attribute file with respective zarr attribute files - children.remove(self._group_meta_key) - children.append(zarr_group_meta_key) - if self._contains_attrs(path): - children.append(zarr_attrs_key) - return sorted(children) - else: - return children - - def _load_n5_attrs(self, path: str): - try: - s = super().__getitem__(path) - return json_loads(s) - except KeyError: - return {} - - def _is_group(self, path: Optional[str]): - if path is None: - attrs_key = self._attrs_key - else: - attrs_key = os.path.join(path, self._attrs_key) - - n5_attrs = self._load_n5_attrs(attrs_key) - return len(n5_attrs) > 0 and "dimensions" not in n5_attrs - - def _is_array(self, path: Optional[str]): - if path is None: - attrs_key = self._attrs_key - else: - attrs_key = os.path.join(path, self._attrs_key) - - return "dimensions" in self._load_n5_attrs(attrs_key) - - def _contains_attrs(self, path: Optional[str]): - if path is None: - attrs_key = self._attrs_key - else: - if not path.endswith(self._attrs_key): - attrs_key = os.path.join(path, self._attrs_key) - else: - attrs_key = path - - attrs = attrs_to_zarr(self._load_n5_attrs(attrs_key)) - return len(attrs) > 0 - - -def is_chunk_key(key: str): - rv = False - segments = list(key.split("/")) - if segments: - last_segment = segments[-1] - rv = bool(_prog_ckey.match(last_segment)) - return rv - - -def invert_chunk_coords(key: str): - segments = list(key.split("/")) - if segments: - last_segment = segments[-1] - if _prog_ckey.match(last_segment): - coords = list(last_segment.split(".")) - last_segment = "/".join(coords[::-1]) - segments = segments[:-1] + [last_segment] - key = "/".join(segments) - return key - - -def group_metadata_to_n5(group_metadata: Dict[str, Any]) -> Dict[str, Any]: - """Convert group metadata from zarr to N5 format.""" - del group_metadata["zarr_format"] - # TODO: This should only exist at the top-level - group_metadata["n5"] = N5_FORMAT - return group_metadata - - -def group_metadata_to_zarr(group_metadata: Dict[str, Any]) -> Dict[str, Any]: - """Convert group metadata from N5 to zarr format.""" - # This only exists at the top level - group_metadata.pop("n5", None) - group_metadata["zarr_format"] = ZARR_FORMAT - return group_metadata - - -def array_metadata_to_n5(array_metadata: Dict[str, Any], top_level=False) -> Dict[str, Any]: - """Convert array metadata from zarr to N5 format. If the `top_level` keyword argument is True, - then the `N5` : N5_FORMAT key : value pair will be inserted into the metadata.""" - - for f, t in zarr_to_n5_keys: - array_metadata[t] = array_metadata.pop(f) - del array_metadata["zarr_format"] - if top_level: - array_metadata["n5"] = N5_FORMAT - try: - dtype = np.dtype(array_metadata["dataType"]) - except TypeError: - raise TypeError(f"Data type {array_metadata['dataType']} is not supported by N5") - - array_metadata["dataType"] = dtype.name - array_metadata["dimensions"] = array_metadata["dimensions"][::-1] - array_metadata["blockSize"] = array_metadata["blockSize"][::-1] - - if "fill_value" in array_metadata: - if array_metadata["fill_value"] != 0 and array_metadata["fill_value"] is not None: - raise ValueError( - f"""Received fill_value = {array_metadata['fill_value']}, - but N5 only supports fill_value = 0""" - ) - del array_metadata["fill_value"] - - if "order" in array_metadata: - if array_metadata["order"] != "C": - raise ValueError( - f"Received order = {array_metadata['order']}, but N5 only supports order = C" - ) - del array_metadata["order"] - - if "filters" in array_metadata: - if array_metadata["filters"] != [] and array_metadata["filters"] is not None: - raise ValueError("Received filters, but N5 storage does not support zarr filters") - del array_metadata["filters"] - - assert "compression" in array_metadata - compressor_config = array_metadata["compression"] - compressor_config = compressor_config_to_n5(compressor_config) - array_metadata["compression"] = compressor_config - - if "dimension_separator" in array_metadata: - del array_metadata["dimension_separator"] - - return array_metadata - - -def array_metadata_to_zarr( - array_metadata: Dict[str, Any], top_level: bool = False -) -> Dict[str, Any]: - """Convert array metadata from N5 to zarr format. - If the `top_level` keyword argument is True, then the `N5` key will be removed from metadata""" - for t, f in zarr_to_n5_keys: - array_metadata[t] = array_metadata.pop(f) - if top_level: - array_metadata.pop("n5") - array_metadata["zarr_format"] = ZARR_FORMAT - - array_metadata["shape"] = array_metadata["shape"][::-1] - array_metadata["chunks"] = array_metadata["chunks"][::-1] - array_metadata["fill_value"] = 0 # also if None was requested - array_metadata["order"] = "C" - array_metadata["filters"] = [] - array_metadata["dimension_separator"] = "." - array_metadata["dtype"] = np.dtype(array_metadata["dtype"]).str - - compressor_config = array_metadata["compressor"] - compressor_config = compressor_config_to_zarr(compressor_config) - array_metadata["compressor"] = { - "id": N5ChunkWrapper.codec_id, - "compressor_config": compressor_config, - "dtype": array_metadata["dtype"], - "chunk_shape": array_metadata["chunks"], - } - - return array_metadata - - -def attrs_to_zarr(attrs: Dict[str, Any]) -> Dict[str, Any]: - """Get all zarr attributes from an N5 attributes dictionary (i.e., - all non-keyword attributes).""" - - # remove all N5 keywords - for n5_key in n5_keywords: - if n5_key in attrs: - del attrs[n5_key] - - return attrs - - -def compressor_config_to_n5(compressor_config: Optional[Dict[str, Any]]) -> Dict[str, Any]: - if compressor_config is None: - return {"type": "raw"} - else: - _compressor_config = compressor_config - - # peel wrapper, if present - if _compressor_config["id"] == N5ChunkWrapper.codec_id: - _compressor_config = _compressor_config["compressor_config"] - - codec_id = _compressor_config["id"] - n5_config = {"type": codec_id} - - if codec_id == "bz2": - n5_config["type"] = "bzip2" - n5_config["blockSize"] = _compressor_config["level"] - - elif codec_id == "blosc": - n5_config["cname"] = _compressor_config["cname"] - n5_config["clevel"] = _compressor_config["clevel"] - n5_config["shuffle"] = _compressor_config["shuffle"] - n5_config["blocksize"] = _compressor_config["blocksize"] - - elif codec_id == "lzma": - # Switch to XZ for N5 if we are using the default XZ format. - # Note: 4 is the default, which is lzma.CHECK_CRC64. - if _compressor_config["format"] == 1 and _compressor_config["check"] in [-1, 4]: - n5_config["type"] = "xz" - else: - warnings.warn( - "Not all N5 implementations support lzma compression (yet). You " - "might not be able to open the dataset with another N5 library.", - RuntimeWarning, - ) - n5_config["format"] = _compressor_config["format"] - n5_config["check"] = _compressor_config["check"] - n5_config["filters"] = _compressor_config["filters"] - - # The default is lzma.PRESET_DEFAULT, which is 6. - if _compressor_config["preset"]: - n5_config["preset"] = _compressor_config["preset"] - else: - n5_config["preset"] = 6 - - elif codec_id == "zlib": - n5_config["type"] = "gzip" - n5_config["level"] = _compressor_config["level"] - n5_config["useZlib"] = True - - elif codec_id == "gzip": - n5_config["type"] = "gzip" - n5_config["level"] = _compressor_config["level"] - n5_config["useZlib"] = False - - else: - n5_config.update({k: v for k, v in _compressor_config.items() if k != "type"}) - - return n5_config - - -def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dict[str, Any]]: - codec_id = compressor_config["type"] - zarr_config = {"id": codec_id} - - if codec_id == "bzip2": - zarr_config["id"] = "bz2" - zarr_config["level"] = compressor_config["blockSize"] - - elif codec_id == "blosc": - zarr_config["cname"] = compressor_config["cname"] - zarr_config["clevel"] = compressor_config["clevel"] - zarr_config["shuffle"] = compressor_config["shuffle"] - zarr_config["blocksize"] = compressor_config["blocksize"] - - elif codec_id == "lzma": - zarr_config["format"] = compressor_config["format"] - zarr_config["check"] = compressor_config["check"] - zarr_config["preset"] = compressor_config["preset"] - zarr_config["filters"] = compressor_config["filters"] - - elif codec_id == "xz": - zarr_config["id"] = "lzma" - zarr_config["format"] = 1 # lzma.FORMAT_XZ - zarr_config["check"] = -1 - zarr_config["preset"] = compressor_config["preset"] - zarr_config["filters"] = None - - elif codec_id == "gzip": - if compressor_config.get("useZlib"): - zarr_config["id"] = "zlib" - zarr_config["level"] = compressor_config["level"] - else: - zarr_config["id"] = "gzip" - zarr_config["level"] = compressor_config["level"] - - elif codec_id == "raw": - return None - - else: - zarr_config.update({k: v for k, v in compressor_config.items() if k != "type"}) - - return zarr_config - - -class N5ChunkWrapper(Codec): # type: ignore[misc] - codec_id = "n5_wrapper" - - def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None): - self.dtype = np.dtype(dtype) - self.chunk_shape = tuple(chunk_shape) - # is the dtype a little endian format? - self._little_endian = self.dtype.byteorder == "<" or ( - self.dtype.byteorder == "=" and sys.byteorder == "little" - ) - - if compressor: - if compressor_config is not None: - raise ValueError("Only one of compressor_config or compressor should be given.") - compressor_config = compressor.get_config() - - if compressor_config is None and compressor is None or compressor_config["id"] == "raw": - self.compressor_config = None - self._compressor = None - else: - self._compressor = get_codec(compressor_config) - self.compressor_config = self._compressor.get_config() - - def get_config(self): - config = {"id": self.codec_id, "compressor_config": self.compressor_config} - return config - - def encode(self, chunk): - assert chunk.flags.c_contiguous - - header = self._create_header(chunk) - chunk = self._to_big_endian(chunk) - - if self._compressor: - return header + self._compressor.encode(chunk) - else: - return header + chunk.tobytes(order="A") - - def decode(self, chunk, out=None) -> bytes: - len_header, chunk_shape = self._read_header(chunk) - chunk = chunk[len_header:] - - if out is not None: - # out should only be used if we read a complete chunk - assert chunk_shape == self.chunk_shape, "Expected chunk of shape {}, found {}".format( - self.chunk_shape, chunk_shape - ) - - if self._compressor: - self._compressor.decode(chunk, out) - else: - ndarray_copy(chunk, out) - - # we can byteswap in-place - if self._little_endian: - out.byteswap(True) - - return out - - else: - if self._compressor: - chunk = self._compressor.decode(chunk) - - # more expensive byteswap - chunk = self._from_big_endian(chunk) - - # read partial chunk - if chunk_shape != self.chunk_shape: - chunk = np.frombuffer(chunk, dtype=self.dtype) - chunk = chunk.reshape(chunk_shape) - complete_chunk = np.zeros(self.chunk_shape, dtype=self.dtype) - target_slices = tuple(slice(0, s) for s in chunk_shape) - complete_chunk[target_slices] = chunk - chunk = complete_chunk - - return chunk - - @staticmethod - def _create_header(chunk): - mode = struct.pack(">H", 0) - num_dims = struct.pack(">H", len(chunk.shape)) - shape = b"".join(struct.pack(">I", d) for d in chunk.shape[::-1]) - - return mode + num_dims + shape - - @staticmethod - def _read_header(chunk): - num_dims = struct.unpack(">H", chunk[2:4])[0] - shape = tuple( - struct.unpack(">I", chunk[i : i + 4])[0] for i in range(4, num_dims * 4 + 4, 4) - )[::-1] - - len_header = 4 + num_dims * 4 - - return len_header, shape - - def _to_big_endian(self, data): - # assumes data is ndarray - - if self._little_endian: - return data.byteswap() - return data - - def _from_big_endian(self, data): - # assumes data is byte array in big endian - - if not self._little_endian: - return data - - a = np.frombuffer(data, self.dtype.newbyteorder(">")) - return a.astype(self.dtype) - - -register_codec(N5ChunkWrapper, N5ChunkWrapper.codec_id) diff --git a/src/zarr/v2/storage.py b/src/zarr/v2/storage.py deleted file mode 100644 index 67240e520d..0000000000 --- a/src/zarr/v2/storage.py +++ /dev/null @@ -1,2822 +0,0 @@ -"""This module contains storage classes for use with Zarr arrays and groups. - -Note that any object implementing the :class:`MutableMapping` interface from the -:mod:`collections` module in the Python standard library can be used as a Zarr -array store, as long as it accepts string (str) keys and bytes values. - -In addition to the :class:`MutableMapping` interface, store classes may also implement -optional methods `listdir` (list members of a "directory") and `rmdir` (remove all -members of a "directory"). These methods should be implemented if the store class is -aware of the hierarchical organisation of resources within the store and can provide -efficient implementations. If these methods are not available, Zarr will fall back to -slower implementations that work via the :class:`MutableMapping` interface. Store -classes may also optionally implement a `rename` method (rename all members under a given -path) and a `getsize` method (return the size in bytes of a given value). - -""" - -import atexit -import errno -import glob -import multiprocessing -import operator -import os -import re -import shutil -import sys -import tempfile -import warnings -import zipfile -from collections import OrderedDict -from collections.abc import MutableMapping -from os import scandir -from pickle import PicklingError -from threading import Lock, RLock -from typing import Sequence, Mapping, Optional, Union, List, Tuple, Dict, Any -import uuid -import time - -from numcodecs.compat import ensure_bytes, ensure_text, ensure_contiguous_ndarray_like -from numcodecs.registry import codec_registry -from zarr.v2.context import Context - -from zarr.v2.errors import ( - MetadataError, - BadCompressorError, - ContainsArrayError, - ContainsGroupError, - FSPathExistNotDir, - ReadOnlyError, -) -from zarr.v2.meta import encode_array_metadata, encode_group_metadata -from zarr.v2.util import ( - buffer_size, - json_loads, - nolock, - normalize_chunks, - normalize_dimension_separator, - normalize_dtype, - normalize_fill_value, - normalize_order, - normalize_shape, - normalize_storage_path, - retry_call, - ensure_contiguous_ndarray_or_bytes, -) - -from zarr.v2._storage.absstore import ABSStore # noqa: F401 -from zarr.v2._storage.store import ( # noqa: F401 - _listdir_from_keys, - _rename_from_keys, - _rmdir_from_keys, - _path_to_prefix, - _prefix_to_array_key, - _prefix_to_group_key, - array_meta_key, - attrs_key, - group_meta_key, - DEFAULT_ZARR_VERSION, - BaseStore, - Store, -) - -__doctest_requires__ = { - ("RedisStore", "RedisStore.*"): ["redis"], - ("MongoDBStore", "MongoDBStore.*"): ["pymongo"], - ("LRUStoreCache", "LRUStoreCache.*"): ["s3fs"], -} - - -try: - # noinspection PyUnresolvedReferences - from zarr.v2.codecs import Blosc - - default_compressor = Blosc() -except ImportError: # pragma: no cover - from zarr.v2.codecs import Zlib - - default_compressor = Zlib() - - -Path = Union[str, bytes, None] -# allow MutableMapping for backwards compatibility -StoreLike = Union[BaseStore, MutableMapping[str, Any]] - - -def contains_array(store: StoreLike, path: Path = None) -> bool: - """Return True if the store contains an array at the given logical path.""" - path = normalize_storage_path(path) - prefix = _path_to_prefix(path) - key = _prefix_to_array_key(store, prefix) - return key in store - - -def contains_group(store: StoreLike, path: Path = None, explicit_only=True) -> bool: - """Return True if the store contains a group at the given logical path.""" - path = normalize_storage_path(path) - prefix = _path_to_prefix(path) - key = _prefix_to_group_key(store, prefix) - return key in store - - -def normalize_store_arg(store: Any, storage_options=None, mode="r") -> BaseStore: - if store is None: - store = KVStore(dict()) - return store - if isinstance(store, os.PathLike): - store = os.fspath(store) - if FSStore._fsspec_installed(): - import fsspec - - if isinstance(store, fsspec.FSMap): - return FSStore( - store.root, - fs=store.fs, - mode=mode, - check=store.check, - create=store.create, - missing_exceptions=store.missing_exceptions, - **(storage_options or {}), - ) - if isinstance(store, str): - if "://" in store or "::" in store: - return FSStore(store, mode=mode, **(storage_options or {})) - elif storage_options: - raise ValueError("storage_options passed with non-fsspec path") - if store.endswith(".zip"): - return ZipStore(store, mode=mode) - elif store.endswith(".n5"): - from zarr.v2.n5 import N5Store - - return N5Store(store) - else: - return DirectoryStore(store) - else: - store = Store._ensure_store(store) - return store - - -def rmdir(store: StoreLike, path: Path = None): - """Remove all items under the given path. If `store` provides a `rmdir` method, - this will be called, otherwise will fall back to implementation via the - `Store` interface.""" - path = normalize_storage_path(path) - if hasattr(store, "rmdir") and store.is_erasable(): # type: ignore - # pass through - store.rmdir(path) - else: - # slow version, delete one key at a time - _rmdir_from_keys(store, path) - - -def rename(store: Store, src_path: Path, dst_path: Path): - """Rename all items under the given path. If `store` provides a `rename` method, - this will be called, otherwise will fall back to implementation via the - `Store` interface.""" - src_path = normalize_storage_path(src_path) - dst_path = normalize_storage_path(dst_path) - if hasattr(store, "rename"): - # pass through - store.rename(src_path, dst_path) - else: - # slow version, delete one key at a time - _rename_from_keys(store, src_path, dst_path) - - -def listdir(store: BaseStore, path: Path = None): - """Obtain a directory listing for the given path. If `store` provides a `listdir` - method, this will be called, otherwise will fall back to implementation via the - `MutableMapping` interface.""" - path = normalize_storage_path(path) - if hasattr(store, "listdir"): - # pass through - return store.listdir(path) - else: - # slow version, iterate through all keys - warnings.warn( - f"Store {store} has no `listdir` method. From zarr 2.9 onwards " - "may want to inherit from `Store`.", - stacklevel=2, - ) - return _listdir_from_keys(store, path) - - -def _getsize(store: BaseStore, path: Path = None) -> int: - # compute from size of values - if isinstance(path, str) and path in store: - v = store[path] - size = buffer_size(v) - else: - path = "" if path is None else normalize_storage_path(path) - size = 0 - - members = listdir(store, path) - prefix = _path_to_prefix(path) - members = [prefix + k for k in members] - for k in members: - try: - v = store[k] - except KeyError: - pass - else: - try: - size += buffer_size(v) - except TypeError: - return -1 - return size - - -def getsize(store: BaseStore, path: Path = None) -> int: - """Compute size of stored items for a given path. If `store` provides a `getsize` - method, this will be called, otherwise will return -1.""" - if hasattr(store, "getsize"): - # pass through - path = normalize_storage_path(path) - return store.getsize(path) - elif isinstance(store, MutableMapping): - return _getsize(store, path) - else: - return -1 - - -def _require_parent_group( - path: Optional[str], - store: StoreLike, - chunk_store: Optional[StoreLike], - overwrite: bool, -): - # assume path is normalized - if path: - segments = path.split("/") - for i in range(len(segments)): - p = "/".join(segments[:i]) - if contains_array(store, p): - _init_group_metadata(store, path=p, chunk_store=chunk_store, overwrite=overwrite) - elif not contains_group(store, p): - _init_group_metadata(store, path=p, chunk_store=chunk_store) - - -def init_array( - store: StoreLike, - shape: Union[int, Tuple[int, ...]], - chunks: Union[bool, int, Tuple[int, ...]] = True, - dtype=None, - compressor="default", - fill_value=None, - order: str = "C", - overwrite: bool = False, - path: Optional[Path] = None, - chunk_store: Optional[StoreLike] = None, - filters=None, - object_codec=None, - dimension_separator=None, - storage_transformers=(), -): - """Initialize an array store with the given configuration. Note that this is a low-level - function and there should be no need to call this directly from user code. - - Parameters - ---------- - store : Store - A mapping that supports string keys and bytes-like values. - shape : int or tuple of ints - Array shape. - chunks : bool, int or tuple of ints, optional - Chunk shape. If True, will be guessed from `shape` and `dtype`. If - False, will be set to `shape`, i.e., single chunk for the whole array. - dtype : string or dtype, optional - NumPy dtype. - compressor : Codec, optional - Primary compressor. - fill_value : object - Default value to use for uninitialized portions of the array. - order : {'C', 'F'}, optional - Memory layout to be used within each chunk. - overwrite : bool, optional - If True, erase all data in `store` prior to initialisation. - path : string, bytes, optional - Path under which array is stored. - chunk_store : Store, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - filters : sequence, optional - Sequence of filters to use to encode chunk data prior to compression. - object_codec : Codec, optional - A codec to encode object arrays, only needed if dtype=object. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Examples - -------- - Initialize an array store:: - - >>> from zarr.v2.storage import init_array, KVStore - >>> store = KVStore(dict()) - >>> init_array(store, shape=(10000, 10000), chunks=(1000, 1000)) - >>> sorted(store.keys()) - ['.zarray'] - - Array metadata is stored as JSON:: - - >>> print(store['.zarray'].decode()) - { - "chunks": [ - 1000, - 1000 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": ">> store = KVStore(dict()) - >>> init_array(store, shape=100000000, chunks=1000000, dtype='i1', path='foo') - >>> sorted(store.keys()) - ['.zgroup', 'foo/.zarray'] - >>> print(store['foo/.zarray'].decode()) - { - "chunks": [ - 1000000 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|i1", - "fill_value": null, - "filters": null, - "order": "C", - "shape": [ - 100000000 - ], - "zarr_format": 2 - } - - Notes - ----- - The initialisation process involves normalising all array metadata, encoding - as JSON and storing under the '.zarray' key. - - """ - - # normalize path - path = normalize_storage_path(path) - - # ensure parent group initialized - - _require_parent_group(path, store=store, chunk_store=chunk_store, overwrite=overwrite) - - if not compressor: - # compatibility with legacy tests using compressor=[] - compressor = None - _init_array_metadata( - store, - shape=shape, - chunks=chunks, - dtype=dtype, - compressor=compressor, - fill_value=fill_value, - order=order, - overwrite=overwrite, - path=path, - chunk_store=chunk_store, - filters=filters, - object_codec=object_codec, - dimension_separator=dimension_separator, - storage_transformers=storage_transformers, - ) - - -def _init_array_metadata( - store: StoreLike, - shape, - chunks=None, - dtype=None, - compressor="default", - fill_value=None, - order="C", - overwrite=False, - path: Optional[str] = None, - chunk_store: Optional[StoreLike] = None, - filters=None, - object_codec=None, - dimension_separator=None, - storage_transformers=(), -): - path = normalize_storage_path(path) - - # guard conditions - if overwrite: - # attempt to delete any pre-existing array in store - rmdir(store, path) - if chunk_store is not None: - rmdir(chunk_store, path) - - if not overwrite: - if contains_array(store, path): - raise ContainsArrayError(path) - if contains_group(store, path, explicit_only=False): - raise ContainsGroupError(path) - - # normalize metadata - dtype, object_codec = normalize_dtype(dtype, object_codec) - shape = normalize_shape(shape) + dtype.shape - dtype = dtype.base - chunks = normalize_chunks(chunks, shape, dtype.itemsize) - order = normalize_order(order) - fill_value = normalize_fill_value(fill_value, dtype) - - # optional array metadata - if dimension_separator is None: - dimension_separator = getattr(store, "_dimension_separator", None) - dimension_separator = normalize_dimension_separator(dimension_separator) - - # compressor prep - if shape == (): - # no point in compressing a 0-dimensional array, only a single value - compressor = None - elif compressor == "none": - # compatibility - compressor = None - elif compressor == "default": - compressor = default_compressor - - # obtain compressor config - compressor_config = None - if compressor: - try: - compressor_config = compressor.get_config() - except AttributeError as e: - raise BadCompressorError(compressor) from e - - # obtain filters config - if filters: - # TODO: filters was removed from the metadata in v3 - # raise error here if store_version > 2? - filters_config = [f.get_config() for f in filters] - else: - filters_config = [] - - # deal with object encoding - if dtype.hasobject: - if object_codec is None: - if not filters: - # there are no filters so we can be sure there is no object codec - raise ValueError("missing object_codec for object array") - else: - # one of the filters may be an object codec, issue a warning rather - # than raise an error to maintain backwards-compatibility - warnings.warn( - "missing object_codec for object array; this will raise a " - "ValueError in version 3.0", - FutureWarning, - ) - else: - filters_config.insert(0, object_codec.get_config()) - elif object_codec is not None: - warnings.warn("an object_codec is only needed for object arrays") - - # use null to indicate no filters - if not filters_config: - filters_config = None # type: ignore - - # initialize metadata - _compressor = compressor_config - meta = dict( - shape=shape, - compressor=_compressor, - fill_value=fill_value, - dimension_separator=dimension_separator, - ) - - meta.update(dict(chunks=chunks, dtype=dtype, order=order, filters=filters_config)) - assert not storage_transformers - - key = _prefix_to_array_key(store, _path_to_prefix(path)) - if hasattr(store, "_metadata_class"): - store[key] = store._metadata_class.encode_array_metadata(meta) - else: - store[key] = encode_array_metadata(meta) - - -# backwards compatibility -init_store = init_array - - -def init_group( - store: StoreLike, - overwrite: bool = False, - path: Path = None, - chunk_store: Optional[StoreLike] = None, -): - """Initialize a group store. Note that this is a low-level function and there should be no - need to call this directly from user code. - - Parameters - ---------- - store : Store - A mapping that supports string keys and byte sequence values. - overwrite : bool, optional - If True, erase all data in `store` prior to initialisation. - path : string, optional - Path under which array is stored. - chunk_store : Store, optional - Separate storage for chunks. If not provided, `store` will be used - for storage of both chunks and metadata. - - """ - - # normalize path - path = normalize_storage_path(path) - - _require_parent_group(path, store=store, chunk_store=chunk_store, overwrite=overwrite) - - # initialise metadata - _init_group_metadata(store=store, overwrite=overwrite, path=path, chunk_store=chunk_store) - - -def _init_group_metadata( - store: StoreLike, - overwrite: Optional[bool] = False, - path: Optional[str] = None, - chunk_store: Optional[StoreLike] = None, -): - path = normalize_storage_path(path) - - # guard conditions - if overwrite: - # attempt to delete any pre-existing items in store - rmdir(store, path) - if chunk_store is not None: - rmdir(chunk_store, path) - - if not overwrite: - if contains_array(store, path): - raise ContainsArrayError(path) - elif contains_group(store, path): - raise ContainsGroupError(path) - - # initialize metadata - # N.B., currently no metadata properties are needed, however there may - # be in future - meta: dict[str, Any] = {} - key = _prefix_to_group_key(store, _path_to_prefix(path)) - if hasattr(store, "_metadata_class"): - store[key] = store._metadata_class.encode_group_metadata(meta) - else: - store[key] = encode_group_metadata(meta) - - -def _dict_store_keys(d: dict[str, Any], prefix="", cls=dict): - for k in d.keys(): - v = d[k] - if isinstance(v, cls): - yield from _dict_store_keys(v, prefix + k + "/", cls) - else: - yield prefix + k - - -class KVStore(Store): - """ - This provides a default implementation of a store interface around - a mutable mapping, to avoid having to test stores for presence of methods. - - This, for most methods should just be a pass-through to the underlying KV - store which is likely to expose a MuttableMapping interface, - """ - - def __init__(self, mutablemapping): - self._mutable_mapping = mutablemapping - - def __getitem__(self, key): - return self._mutable_mapping[key] - - def __setitem__(self, key, value): - self._mutable_mapping[key] = value - - def __delitem__(self, key): - del self._mutable_mapping[key] - - def __contains__(self, key): - return key in self._mutable_mapping - - def get(self, key, default=None): - return self._mutable_mapping.get(key, default) - - def values(self): - return self._mutable_mapping.values() - - def __iter__(self): - return iter(self._mutable_mapping) - - def __len__(self): - return len(self._mutable_mapping) - - def __repr__(self): - return f"<{self.__class__.__name__}: \n{self._mutable_mapping!r}\n at {hex(id(self))}>" - - def __eq__(self, other): - if isinstance(other, KVStore): - return self._mutable_mapping == other._mutable_mapping - else: - return NotImplemented - - -class MemoryStore(Store): - """Store class that uses a hierarchy of :class:`KVStore` objects, thus all data - will be held in main memory. - - Examples - -------- - This is the default class used when creating a group. E.g.:: - - >>> import zarr - >>> g = zarr.v2.group() - >>> type(g.store) - - - Note that the default class when creating an array is the built-in - :class:`KVStore` class, i.e.:: - - >>> z = zarr.v2.zeros(100) - >>> type(z.store) - - - Notes - ----- - Safe to write in multiple threads. - - """ - - def __init__(self, root=None, cls=dict, dimension_separator=None): - if root is None: - self.root = cls() - else: - self.root = root - self.cls = cls - self.write_mutex = Lock() - self._dimension_separator = dimension_separator - - def __getstate__(self): - return self.root, self.cls - - def __setstate__(self, state): - root, cls = state - self.__init__(root=root, cls=cls) - - def _get_parent(self, item: str): - parent = self.root - # split the item - segments = item.split("/") - # find the parent container - for k in segments[:-1]: - parent = parent[k] - if not isinstance(parent, self.cls): - raise KeyError(item) - return parent, segments[-1] - - def _require_parent(self, item): - parent = self.root - # split the item - segments = item.split("/") - # require the parent container - for k in segments[:-1]: - try: - parent = parent[k] - except KeyError: - parent[k] = self.cls() - parent = parent[k] - else: - if not isinstance(parent, self.cls): - raise KeyError(item) - return parent, segments[-1] - - def __getitem__(self, item: str): - parent, key = self._get_parent(item) - try: - value = parent[key] - except KeyError: - raise KeyError(item) - else: - if isinstance(value, self.cls): - raise KeyError(item) - else: - return value - - def __setitem__(self, item: str, value): - with self.write_mutex: - parent, key = self._require_parent(item) - value = ensure_bytes(value) - parent[key] = value - - def __delitem__(self, item: str): - with self.write_mutex: - parent, key = self._get_parent(item) - try: - del parent[key] - except KeyError: - raise KeyError(item) - - def __contains__(self, item: str): # type: ignore[override] - try: - parent, key = self._get_parent(item) - value = parent[key] - except KeyError: - return False - else: - return not isinstance(value, self.cls) - - def __eq__(self, other): - return isinstance(other, MemoryStore) and self.root == other.root and self.cls == other.cls - - def keys(self): - yield from _dict_store_keys(self.root, cls=self.cls) - - def __iter__(self): - return self.keys() - - def __len__(self) -> int: - return sum(1 for _ in self.keys()) - - def listdir(self, path: Path = None) -> List[str]: - path = normalize_storage_path(path) - if path: - try: - parent, key = self._get_parent(path) - value = parent[key] - except KeyError: - return [] - else: - value = self.root - if isinstance(value, self.cls): - return sorted(value.keys()) - else: - return [] - - def rename(self, src_path: Path, dst_path: Path): - src_path = normalize_storage_path(src_path) - dst_path = normalize_storage_path(dst_path) - - src_parent, src_key = self._get_parent(src_path) - dst_parent, dst_key = self._require_parent(dst_path) - - dst_parent[dst_key] = src_parent.pop(src_key) - - def rmdir(self, path: Path = None): - path = normalize_storage_path(path) - if path: - try: - parent, key = self._get_parent(path) - value = parent[key] - except KeyError: - return - else: - if isinstance(value, self.cls): - del parent[key] - else: - # clear out root - self.root = self.cls() - - def getsize(self, path: Path = None): - path = normalize_storage_path(path) - - # obtain value to return size of - value = None - if path: - try: - parent, key = self._get_parent(path) - value = parent[key] - except KeyError: - pass - else: - value = self.root - - # obtain size of value - if value is None: - return 0 - - elif isinstance(value, self.cls): - # total size for directory - size = 0 - for v in value.values(): - if not isinstance(v, self.cls): - size += buffer_size(v) - return size - - else: - return buffer_size(value) - - def clear(self): - with self.write_mutex: - self.root.clear() - - -class DictStore(MemoryStore): - def __init__(self, *args, **kwargs): - warnings.warn( - "DictStore has been renamed to MemoryStore in 2.4.0 and " - "will be removed in the future. Please use MemoryStore.", - DeprecationWarning, - stacklevel=2, - ) - super().__init__(*args, **kwargs) - - -class DirectoryStore(Store): - """Storage class using directories and files on a standard file system. - - Parameters - ---------- - path : string - Location of directory to use as the root of the storage hierarchy. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-sensitive and - case-insensitive file system. Default value is False. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.v2.DirectoryStore('data/array.zarr') - >>> z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Each chunk of the array is stored as a separate file on the file system, - i.e.:: - - >>> import os - >>> sorted(os.listdir('data/array.zarr')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - - Store a group:: - - >>> store = zarr.v2.DirectoryStore('data/group.zarr') - >>> root = zarr.v2.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - - When storing a group, levels in the group hierarchy will correspond to - directories on the file system, i.e.:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '0.0', '0.1', '1.0', '1.1'] - - Notes - ----- - Atomic writes are used, which means that data are first written to a - temporary file, then moved into place when the write is successfully - completed. Files are only held open while they are being read or written and are - closed immediately afterwards, so there is no need to manually close any files. - - Safe to write in multiple threads or processes. - - """ - - def __init__(self, path, normalize_keys=False, dimension_separator=None): - # guard conditions - path = os.path.abspath(path) - if os.path.exists(path) and not os.path.isdir(path): - raise FSPathExistNotDir(path) - - self.path = path - self.normalize_keys = normalize_keys - self._dimension_separator = dimension_separator - - def _normalize_key(self, key): - return key.lower() if self.normalize_keys else key - - @staticmethod - def _fromfile(fn): - """Read data from a file - - Parameters - ---------- - fn : str - Filepath to open and read from. - - Notes - ----- - Subclasses should overload this method to specify any custom - file reading logic. - """ - with open(fn, "rb") as f: - return f.read() - - @staticmethod - def _tofile(a, fn): - """Write data to a file - - Parameters - ---------- - a : array-like - Data to write into the file. - fn : str - Filepath to open and write to. - - Notes - ----- - Subclasses should overload this method to specify any custom - file writing logic. - """ - with open(fn, mode="wb") as f: - f.write(a) - - def __getitem__(self, key): - key = self._normalize_key(key) - filepath = os.path.join(self.path, key) - if os.path.isfile(filepath): - return self._fromfile(filepath) - else: - raise KeyError(key) - - def __setitem__(self, key, value): - key = self._normalize_key(key) - - # coerce to flat, contiguous array (ideally without copying) - value = ensure_contiguous_ndarray_like(value) - - # destination path for key - file_path = os.path.join(self.path, key) - - # ensure there is no directory in the way - if os.path.isdir(file_path): - shutil.rmtree(file_path) - - # ensure containing directory exists - dir_path, file_name = os.path.split(file_path) - if os.path.isfile(dir_path): - raise KeyError(key) - if not os.path.exists(dir_path): - try: - os.makedirs(dir_path) - except OSError as e: - if e.errno != errno.EEXIST: - raise KeyError(key) - - # write to temporary file - # note we're not using tempfile.NamedTemporaryFile to avoid restrictive file permissions - temp_name = file_name + "." + uuid.uuid4().hex + ".partial" - temp_path = os.path.join(dir_path, temp_name) - try: - self._tofile(value, temp_path) - - # move temporary file into place; - # make several attempts at writing the temporary file to get past - # potential antivirus file locking issues - retry_call(os.replace, (temp_path, file_path), exceptions=(PermissionError,)) - - finally: - # clean up if temp file still exists for whatever reason - if os.path.exists(temp_path): # pragma: no cover - os.remove(temp_path) - - def __delitem__(self, key): - key = self._normalize_key(key) - path = os.path.join(self.path, key) - if os.path.isfile(path): - os.remove(path) - elif os.path.isdir(path): - # include support for deleting directories, even though strictly - # speaking these do not exist as keys in the store - shutil.rmtree(path) - else: - raise KeyError(key) - - def __contains__(self, key): - key = self._normalize_key(key) - file_path = os.path.join(self.path, key) - return os.path.isfile(file_path) - - def __eq__(self, other): - return isinstance(other, DirectoryStore) and self.path == other.path - - def keys(self): - if os.path.exists(self.path): - yield from self._keys_fast(self.path) - - @staticmethod - def _keys_fast(path, walker=os.walk): - for dirpath, _, filenames in walker(path): - dirpath = os.path.relpath(dirpath, path) - if dirpath == os.curdir: - for f in filenames: - yield f - else: - dirpath = dirpath.replace("\\", "/") - for f in filenames: - yield "/".join((dirpath, f)) - - def __iter__(self): - return self.keys() - - def __len__(self): - return sum(1 for _ in self.keys()) - - def dir_path(self, path=None): - store_path = normalize_storage_path(path) - dir_path = self.path - if store_path: - dir_path = os.path.join(dir_path, store_path) - return dir_path - - def listdir(self, path=None): - return ( - self._nested_listdir(path) - if self._dimension_separator == "/" - else self._flat_listdir(path) - ) - - def _flat_listdir(self, path=None): - dir_path = self.dir_path(path) - if os.path.isdir(dir_path): - return sorted(os.listdir(dir_path)) - else: - return [] - - def _nested_listdir(self, path=None): - children = self._flat_listdir(path=path) - if array_meta_key in children: - # special handling of directories containing an array to map nested chunk - # keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and os.path.isdir(entry_path): - for dir_path, _, file_names in os.walk(entry_path): - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path + os.path.sep)[1] - new_children.append( - rel_path.replace(os.path.sep, self._dimension_separator or ".") - ) - else: - new_children.append(entry) - return sorted(new_children) - else: - return children - - def rename(self, src_path, dst_path): - store_src_path = normalize_storage_path(src_path) - store_dst_path = normalize_storage_path(dst_path) - - dir_path = self.path - - src_path = os.path.join(dir_path, store_src_path) - dst_path = os.path.join(dir_path, store_dst_path) - - os.renames(src_path, dst_path) - - def rmdir(self, path=None): - store_path = normalize_storage_path(path) - dir_path = self.path - if store_path: - dir_path = os.path.join(dir_path, store_path) - if os.path.isdir(dir_path): - shutil.rmtree(dir_path) - - def getsize(self, path=None): - store_path = normalize_storage_path(path) - fs_path = self.path - if store_path: - fs_path = os.path.join(fs_path, store_path) - if os.path.isfile(fs_path): - return os.path.getsize(fs_path) - elif os.path.isdir(fs_path): - size = 0 - for child in scandir(fs_path): - if child.is_file(): - size += child.stat().st_size - return size - else: - return 0 - - def clear(self): - shutil.rmtree(self.path) - - -def atexit_rmtree(path, isdir=os.path.isdir, rmtree=shutil.rmtree): # pragma: no cover - """Ensure directory removal at interpreter exit.""" - if isdir(path): - rmtree(path) - - -# noinspection PyShadowingNames -def atexit_rmglob( - path, - glob=glob.glob, - isdir=os.path.isdir, - isfile=os.path.isfile, - remove=os.remove, - rmtree=shutil.rmtree, -): # pragma: no cover - """Ensure removal of multiple files at interpreter exit.""" - for p in glob(path): - if isfile(p): - remove(p) - elif isdir(p): - rmtree(p) - - -class FSStore(Store): - """Wraps an fsspec.FSMap to give access to arbitrary filesystems - - Requires that ``fsspec`` is installed, as well as any additional - requirements for the protocol chosen. - - Parameters - ---------- - url : str - The destination to map. If no fs is provided, should include protocol - and path, like "s3://bucket/root". If an fs is provided, can be a path - within that filesystem, like "bucket/root" - normalize_keys : bool - key_separator : str - public API for accessing dimension_separator. Never `None` - See dimension_separator for more information. - mode : str - "w" for writable, "r" for read-only - exceptions : list of Exception subclasses - When accessing data, any of these exceptions will be treated - as a missing key - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - fs : fsspec.spec.AbstractFileSystem, optional - An existing filesystem to use for the store. - check : bool, optional - If True, performs a touch at the root location, to check for write access. - Passed to `fsspec.mapping.FSMap` constructor. - create : bool, optional - If True, performs a mkdir at the rool location. - Passed to `fsspec.mapping.FSMap` constructor. - missing_exceptions : sequence of Exceptions, optional - Exceptions classes to associate with missing files. - Passed to `fsspec.mapping.FSMap` constructor. - storage_options : passed to the fsspec implementation. Cannot be used - together with fs. - """ - - _array_meta_key = array_meta_key - _group_meta_key = group_meta_key - _attrs_key = attrs_key - - def __init__( - self, - url, - normalize_keys=False, - key_separator=None, - mode="w", - exceptions=(KeyError, PermissionError, IOError), - dimension_separator=None, - fs=None, - check=False, - create=False, - missing_exceptions=None, - **storage_options, - ): - if not self._fsspec_installed(): # pragma: no cover - raise ImportError("`fsspec` is required to use zarr's FSStore") - import fsspec - - mapper_options = {"check": check, "create": create} - # https://github.com/zarr-developers/zarr-python/pull/911#discussion_r841926292 - # Some fsspec implementations don't accept missing_exceptions. - # This is a workaround to avoid passing it in the most common scenarios. - # Remove this and add missing_exceptions to mapper_options when fsspec is released. - if missing_exceptions is not None: - mapper_options["missing_exceptions"] = missing_exceptions # pragma: no cover - - if fs is None: - protocol, _ = fsspec.core.split_protocol(url) - # set auto_mkdir to True for local file system - if protocol in (None, "file") and not storage_options.get("auto_mkdir"): - storage_options["auto_mkdir"] = True - self.map = fsspec.get_mapper(url, **{**mapper_options, **storage_options}) - self.fs = self.map.fs # for direct operations - self.path = self.fs._strip_protocol(url) - else: - if storage_options: - raise ValueError("Cannot specify both fs and storage_options") - self.fs = fs - self.path = self.fs._strip_protocol(url) - self.map = self.fs.get_mapper(self.path, **mapper_options) - - self.normalize_keys = normalize_keys - self.mode = mode - self.exceptions = exceptions - # For backwards compatibility. Guaranteed to be non-None - if key_separator is not None: - dimension_separator = key_separator - - self.key_separator = dimension_separator - self._default_key_separator() - - # Pass attributes to array creation - self._dimension_separator = dimension_separator - - def _default_key_separator(self): - if self.key_separator is None: - self.key_separator = "." - - def _normalize_key(self, key): - key = normalize_storage_path(key).lstrip("/") - if key: - *bits, end = key.split("/") - - if end not in (self._array_meta_key, self._group_meta_key, self._attrs_key): - end = end.replace(".", self.key_separator) - key = "/".join(bits + [end]) - - return key.lower() if self.normalize_keys else key - - def getitems( - self, keys: Sequence[str], *, contexts: Mapping[str, Context] - ) -> Mapping[str, Any]: - keys_transformed = [self._normalize_key(key) for key in keys] - results = self.map.getitems(keys_transformed, on_error="omit") - # The function calling this method may not recognize the transformed keys - # So we send the values returned by self.map.getitems back into the original key space. - return {keys[keys_transformed.index(rk)]: rv for rk, rv in results.items()} - - def __getitem__(self, key): - key = self._normalize_key(key) - try: - return self.map[key] - except self.exceptions as e: - raise KeyError(key) from e - - def setitems(self, values): - if self.mode == "r": - raise ReadOnlyError - - # Normalize keys and make sure the values are bytes - values = { - self._normalize_key(key): ensure_contiguous_ndarray_or_bytes(val) - for key, val in values.items() - } - self.map.setitems(values) - - def __setitem__(self, key, value): - if self.mode == "r": - raise ReadOnlyError - key = self._normalize_key(key) - value = ensure_contiguous_ndarray_or_bytes(value) - path = self.dir_path(key) - try: - if self.fs.isdir(path): - self.fs.rm(path, recursive=True) - self.map[key] = value - self.fs.invalidate_cache(self.fs._parent(path)) - except self.exceptions as e: - raise KeyError(key) from e - - def __delitem__(self, key): - if self.mode == "r": - raise ReadOnlyError - key = self._normalize_key(key) - path = self.dir_path(key) - if self.fs.isdir(path): - self.fs.rm(path, recursive=True) - else: - del self.map[key] - - def delitems(self, keys): - if self.mode == "r": - raise ReadOnlyError - # only remove the keys that exist in the store - nkeys = [self._normalize_key(key) for key in keys if key in self] - # rm errors if you pass an empty collection - if len(nkeys) > 0: - self.map.delitems(nkeys) - - def __contains__(self, key): - key = self._normalize_key(key) - return key in self.map - - def __eq__(self, other): - return type(self) is type(other) and self.map == other.map and self.mode == other.mode - - def keys(self): - return iter(self.map) - - def __iter__(self): - return self.keys() - - def __len__(self): - return len(list(self.keys())) - - def dir_path(self, path=None): - store_path = normalize_storage_path(path) - return self.map._key_to_str(store_path) - - def listdir(self, path=None): - dir_path = self.dir_path(path) - try: - children = sorted( - p.rstrip("/").rsplit("/", 1)[-1] for p in self.fs.ls(dir_path, detail=False) - ) - if self.key_separator != "/": - return children - else: - if self._array_meta_key in children: - # special handling of directories containing an array to map nested chunk - # keys back to standard chunk keys - new_children = [] - root_path = self.dir_path(path) - for entry in children: - entry_path = os.path.join(root_path, entry) - if _prog_number.match(entry) and self.fs.isdir(entry_path): - for file_name in self.fs.find(entry_path): - file_path = os.path.join(dir_path, file_name) - rel_path = file_path.split(root_path)[1] - rel_path = rel_path.lstrip("/") - new_children.append(rel_path.replace("/", ".")) - else: - new_children.append(entry) - return sorted(new_children) - else: - return children - except OSError: - return [] - - def rmdir(self, path=None): - if self.mode == "r": - raise ReadOnlyError - store_path = self.dir_path(path) - if self.fs.isdir(store_path): - self.fs.rm(store_path, recursive=True) - - def getsize(self, path=None): - store_path = self.dir_path(path) - return self.fs.du(store_path, True, True) - - def clear(self): - if self.mode == "r": - raise ReadOnlyError - self.map.clear() - - @classmethod - def _fsspec_installed(cls): - """Returns true if fsspec is installed""" - import importlib.util - - return importlib.util.find_spec("fsspec") is not None - - -class TempStore(DirectoryStore): - """Directory store using a temporary directory for storage. - - Parameters - ---------- - suffix : string, optional - Suffix for the temporary directory name. - prefix : string, optional - Prefix for the temporary directory name. - dir : string, optional - Path to parent directory in which to create temporary directory. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-sensitive and - case-insensitive file system. Default value is False. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - """ - - # noinspection PyShadowingBuiltins - def __init__( - self, suffix="", prefix="zarr", dir=None, normalize_keys=False, dimension_separator=None - ): - path = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir) - atexit.register(atexit_rmtree, path) - super().__init__(path, normalize_keys=normalize_keys) - - -_prog_ckey = re.compile(r"^(\d+)(\.\d+)+$") -_prog_number = re.compile(r"^\d+$") - - -class NestedDirectoryStore(DirectoryStore): - """Storage class using directories and files on a standard file system, with - special handling for chunk keys so that chunk files for multidimensional - arrays are stored in a nested directory tree. - - Parameters - ---------- - path : string - Location of directory to use as the root of the storage hierarchy. - normalize_keys : bool, optional - If True, all store keys will be normalized to use lower case characters - (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be - useful to avoid potential discrepancies between case-sensitive and - case-insensitive file system. Default value is False. - dimension_separator : {'/'}, optional - Separator placed between the dimensions of a chunk. - Only supports "/" unlike other implementations. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.v2.NestedDirectoryStore('data/array.zarr') - >>> z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - - Each chunk of the array is stored as a separate file on the file system, - note the multiple directory levels used for the chunk files:: - - >>> import os - >>> sorted(os.listdir('data/array.zarr')) - ['.zarray', '0', '1'] - >>> sorted(os.listdir('data/array.zarr/0')) - ['0', '1'] - >>> sorted(os.listdir('data/array.zarr/1')) - ['0', '1'] - - Store a group:: - - >>> store = zarr.v2.NestedDirectoryStore('data/group.zarr') - >>> root = zarr.v2.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - - When storing a group, levels in the group hierarchy will correspond to - directories on the file system, i.e.:: - - >>> sorted(os.listdir('data/group.zarr')) - ['.zgroup', 'foo'] - >>> sorted(os.listdir('data/group.zarr/foo')) - ['.zgroup', 'bar'] - >>> sorted(os.listdir('data/group.zarr/foo/bar')) - ['.zarray', '0', '1'] - >>> sorted(os.listdir('data/group.zarr/foo/bar/0')) - ['0', '1'] - >>> sorted(os.listdir('data/group.zarr/foo/bar/1')) - ['0', '1'] - - Notes - ----- - The :class:`DirectoryStore` class stores all chunk files for an array - together in a single directory. On some file systems, the potentially large - number of files in a single directory can cause performance issues. The - :class:`NestedDirectoryStore` class provides an alternative where chunk - files for multidimensional arrays will be organised into a directory - hierarchy, thus reducing the number of files in any one directory. - - Safe to write in multiple threads or processes. - - """ - - def __init__(self, path, normalize_keys=False, dimension_separator="/"): - super().__init__(path, normalize_keys=normalize_keys) - if dimension_separator is None: - dimension_separator = "/" - elif dimension_separator != "/": - raise ValueError("NestedDirectoryStore only supports '/' as dimension_separator") - self._dimension_separator = dimension_separator - - def __eq__(self, other): - return isinstance(other, NestedDirectoryStore) and self.path == other.path - - -# noinspection PyPep8Naming -class ZipStore(Store): - """Storage class using a Zip file. - - Parameters - ---------- - path : string - Location of file. - compression : integer, optional - Compression method to use when writing to the archive. - allowZip64 : bool, optional - If True (the default) will create ZIP files that use the ZIP64 - extensions when the zipfile is larger than 2 GiB. If False - will raise an exception when the ZIP file would require ZIP64 - extensions. - mode : string, optional - One of 'r' to read an existing file, 'w' to truncate and write a new - file, 'a' to append to an existing file, or 'x' to exclusively create - and write a new file. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.v2.ZipStore('data/array.zip', mode='w') - >>> z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.v2.ZipStore('data/group.zip', mode='w') - >>> root = zarr.v2.group(store=store) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - - After modifying a ZipStore, the ``close()`` method must be called, otherwise - essential data will not be written to the underlying Zip file. The ZipStore - class also supports the context manager protocol, which ensures the ``close()`` - method is called on leaving the context, e.g.:: - - >>> with zarr.v2.ZipStore('data/array.zip', mode='w') as store: - ... z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store) - ... z[...] = 42 - ... # no need to call store.close() - - Notes - ----- - Each chunk of an array is stored as a separate entry in the Zip file. Note - that Zip files do not provide any way to remove or replace existing entries. - If an attempt is made to replace an entry, then a warning is generated by - the Python standard library about a duplicate Zip file entry. This can be - triggered if you attempt to write data to a Zarr array more than once, - e.g.:: - - >>> store = zarr.v2.ZipStore('data/example.zip', mode='w') - >>> z = zarr.v2.zeros(100, chunks=10, store=store) - >>> # first write OK - ... z[...] = 42 - >>> # second write generates warnings - ... z[...] = 42 # doctest: +SKIP - >>> store.close() - - This can also happen in a more subtle situation, where data are written only - once to a Zarr array, but the write operations are not aligned with chunk - boundaries, e.g.:: - - >>> store = zarr.v2.ZipStore('data/example.zip', mode='w') - >>> z = zarr.v2.zeros(100, chunks=10, store=store) - >>> z[5:15] = 42 - >>> # write overlaps chunk previously written, generates warnings - ... z[15:25] = 42 # doctest: +SKIP - - To avoid creating duplicate entries, only write data once, and align writes - with chunk boundaries. This alignment is done automatically if you call - ``z[...] = ...`` or create an array from existing data via :func:`zarr.v2.array`. - - Alternatively, use a :class:`DirectoryStore` when writing the data, then - manually Zip the directory and use the Zip file for subsequent reads. - Take note that the files in the Zip file must be relative to the root of the - Zarr archive. You may find it easier to create such a Zip file with ``7z``, e.g.:: - - 7z a -tzip archive.zarr.v2.zip archive.zarr/. - - Safe to write in multiple threads but not in multiple processes. - - """ - - _erasable = False - - def __init__( - self, - path, - compression=zipfile.ZIP_STORED, - allowZip64=True, - mode="a", - dimension_separator=None, - ): - # store properties - path = os.path.abspath(path) - self.path = path - self.compression = compression - self.allowZip64 = allowZip64 - self.mode = mode - self._dimension_separator = dimension_separator - - # Current understanding is that zipfile module in stdlib is not thread-safe, - # and so locking is required for both read and write. However, this has not - # been investigated in detail, perhaps no lock is needed if mode='r'. - self.mutex = RLock() - - # open zip file - self.zf = zipfile.ZipFile(path, mode=mode, compression=compression, allowZip64=allowZip64) - - def __getstate__(self): - self.flush() - return self.path, self.compression, self.allowZip64, self.mode - - def __setstate__(self, state): - path, compression, allowZip64, mode = state - # if initially opened with mode 'w' or 'x', re-open in mode 'a' so file doesn't - # get clobbered - if mode in "wx": - mode = "a" - self.__init__(path=path, compression=compression, allowZip64=allowZip64, mode=mode) - - def close(self): - """Closes the underlying zip file, ensuring all records are written.""" - with self.mutex: - self.zf.close() - - def flush(self): - """Closes the underlying zip file, ensuring all records are written, - then re-opens the file for further modifications.""" - if self.mode != "r": - with self.mutex: - self.zf.close() - # N.B., re-open with mode 'a' regardless of initial mode so we don't wipe - # what's been written - self.zf = zipfile.ZipFile( - self.path, mode="a", compression=self.compression, allowZip64=self.allowZip64 - ) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def __getitem__(self, key): - with self.mutex: - with self.zf.open(key) as f: # will raise KeyError - return f.read() - - def __setitem__(self, key, value): - if self.mode == "r": - raise ReadOnlyError - value = ensure_contiguous_ndarray_like(value).view("u1") - with self.mutex: - # writestr(key, value) writes with default permissions from - # zipfile (600) that are too restrictive, build ZipInfo for - # the key to work around limitation - keyinfo = zipfile.ZipInfo(filename=key, date_time=time.localtime(time.time())[:6]) - keyinfo.compress_type = self.compression - if keyinfo.filename[-1] == os.sep: - keyinfo.external_attr = 0o40775 << 16 # drwxrwxr-x - keyinfo.external_attr |= 0x10 # MS-DOS directory flag - else: - keyinfo.external_attr = 0o644 << 16 # ?rw-r--r-- - - self.zf.writestr(keyinfo, value) - - def __delitem__(self, key): - raise NotImplementedError - - def __eq__(self, other): - return ( - isinstance(other, ZipStore) - and self.path == other.path - and self.compression == other.compression - and self.allowZip64 == other.allowZip64 - ) - - def keylist(self): - with self.mutex: - return sorted(self.zf.namelist()) - - def keys(self): - yield from self.keylist() - - def __iter__(self): - return self.keys() - - def __len__(self): - return sum(1 for _ in self.keys()) - - def __contains__(self, key): - try: - with self.mutex: - self.zf.getinfo(key) - except KeyError: - return False - else: - return True - - def listdir(self, path=None): - path = normalize_storage_path(path) - return _listdir_from_keys(self, path) - - def getsize(self, path=None): - path = normalize_storage_path(path) - with self.mutex: - children = self.listdir(path) - if children: - size = 0 - for child in children: - if path: - name = path + "/" + child - else: - name = child - try: - info = self.zf.getinfo(name) - except KeyError: - pass - else: - size += info.compress_size - return size - elif path: - try: - info = self.zf.getinfo(path) - return info.compress_size - except KeyError: - return 0 - else: - return 0 - - def clear(self): - if self.mode == "r": - raise ReadOnlyError - with self.mutex: - self.close() - os.remove(self.path) - self.zf = zipfile.ZipFile( - self.path, mode=self.mode, compression=self.compression, allowZip64=self.allowZip64 - ) - - -def migrate_1to2(store): - """Migrate array metadata in `store` from Zarr format version 1 to - version 2. - - Parameters - ---------- - store : Store - Store to be migrated. - - Notes - ----- - Version 1 did not support hierarchies, so this migration function will - look for a single array in `store` and migrate the array metadata to - version 2. - - """ - - # migrate metadata - from zarr.v2 import meta_v1 - - meta = meta_v1.decode_metadata(store["meta"]) - del store["meta"] - - # add empty filters - meta["filters"] = None - - # migration compression metadata - compression = meta["compression"] - if compression is None or compression == "none": - compressor_config = None - else: - compression_opts = meta["compression_opts"] - codec_cls = codec_registry[compression] - if isinstance(compression_opts, dict): - compressor = codec_cls(**compression_opts) - else: - compressor = codec_cls(compression_opts) - compressor_config = compressor.get_config() - meta["compressor"] = compressor_config - del meta["compression"] - del meta["compression_opts"] - - # store migrated metadata - if hasattr(store, "_metadata_class"): - store[array_meta_key] = store._metadata_class.encode_array_metadata(meta) - else: - store[array_meta_key] = encode_array_metadata(meta) - - # migrate user attributes - store[attrs_key] = store["attrs"] - del store["attrs"] - - -# noinspection PyShadowingBuiltins -class DBMStore(Store): - """Storage class using a DBM-style database. - - Parameters - ---------- - path : string - Location of database file. - flag : string, optional - Flags for opening the database file. - mode : int - File mode used if a new file is created. - open : function, optional - Function to open the database file. If not provided, :func:`dbm.open` will be - used on Python 3, and :func:`anydbm.open` will be used on Python 2. - write_lock: bool, optional - Use a lock to prevent concurrent writes from multiple threads (True by default). - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk.e - **open_kwargs - Keyword arguments to pass the `open` function. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.v2.DBMStore('data/array.db') - >>> z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.v2.DBMStore('data/group.db') - >>> root = zarr.v2.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - - After modifying a DBMStore, the ``close()`` method must be called, otherwise - essential data may not be written to the underlying database file. The - DBMStore class also supports the context manager protocol, which ensures the - ``close()`` method is called on leaving the context, e.g.:: - - >>> with zarr.v2.DBMStore('data/array.db') as store: - ... z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - ... z[...] = 42 - ... # no need to call store.close() - - A different database library can be used by passing a different function to - the `open` parameter. For example, if the `bsddb3 - `_ package is installed, a - Berkeley DB database can be used:: - - >>> import bsddb3 - >>> store = zarr.v2.DBMStore('data/array.bdb', open=bsddb3.btopen) - >>> z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() - - Notes - ----- - Please note that, by default, this class will use the Python standard - library `dbm.open` function to open the database file (or `anydbm.open` on - Python 2). There are up to three different implementations of DBM-style - databases available in any Python installation, and which one is used may - vary from one system to another. Database file formats are not compatible - between these different implementations. Also, some implementations are - more efficient than others. In particular, the "dumb" implementation will be - the fall-back on many systems, and has very poor performance for some usage - scenarios. If you want to ensure a specific implementation is used, pass the - corresponding open function, e.g., `dbm.gnu.open` to use the GNU DBM - library. - - Safe to write in multiple threads. May be safe to write in multiple processes, - depending on which DBM implementation is being used, although this has not been - tested. - - """ - - def __init__( - self, - path, - flag="c", - mode=0o666, - open=None, - write_lock=True, - dimension_separator=None, - **open_kwargs, - ): - if open is None: - import dbm - - open = dbm.open - path = os.path.abspath(path) - # noinspection PyArgumentList - self.db = open(path, flag, mode, **open_kwargs) - self.path = path - self.flag = flag - self.mode = mode - self.open = open - self.write_lock = write_lock - if write_lock: - # This may not be required as some dbm implementations manage their own - # locks, but err on the side of caution. - self.write_mutex = Lock() - else: - self.write_mutex = nolock - self.open_kwargs = open_kwargs - self._dimension_separator = dimension_separator - - def __getstate__(self): - try: - self.flush() # needed for ndbm - except Exception: - # flush may fail if db has already been closed - pass - return (self.path, self.flag, self.mode, self.open, self.write_lock, self.open_kwargs) - - def __setstate__(self, state): - path, flag, mode, open, write_lock, open_kws = state - if flag[0] == "n": - flag = "c" + flag[1:] # don't clobber an existing database - self.__init__(path=path, flag=flag, mode=mode, open=open, write_lock=write_lock, **open_kws) - - def close(self): - """Closes the underlying database file.""" - if hasattr(self.db, "close"): - with self.write_mutex: - self.db.close() - - def flush(self): - """Synchronizes data to the underlying database file.""" - if self.flag[0] != "r": - with self.write_mutex: - if hasattr(self.db, "sync"): - self.db.sync() - else: # pragma: no cover - # we don't cover this branch anymore as ndbm (oracle) is not packaged - # by conda-forge on non-mac OS: - # https://github.com/conda-forge/staged-recipes/issues/4476 - # fall-back, close and re-open, needed for ndbm - flag = self.flag - if flag[0] == "n": - flag = "c" + flag[1:] # don't clobber an existing database - self.db.close() - # noinspection PyArgumentList - self.db = self.open(self.path, flag, self.mode, **self.open_kwargs) - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def __getitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - return self.db[key] - - def __setitem__(self, key, value): - if isinstance(key, str): - key = key.encode("ascii") - value = ensure_bytes(value) - with self.write_mutex: - self.db[key] = value - - def __delitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - with self.write_mutex: - del self.db[key] - - def __eq__(self, other): - return ( - isinstance(other, DBMStore) - and self.path == other.path - and - # allow flag and mode to differ - self.open == other.open - and self.open_kwargs == other.open_kwargs - ) - - def keys(self): - return (ensure_text(k, "ascii") for k in iter(self.db.keys())) - - def __iter__(self): - return self.keys() - - def __len__(self): - return sum(1 for _ in self.keys()) - - def __contains__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - return key in self.db - - def rmdir(self, path: str = "") -> None: - path = normalize_storage_path(path) - _rmdir_from_keys(self, path) - - -class LMDBStore(Store): - """Storage class using LMDB. Requires the `lmdb `_ - package to be installed. - - - Parameters - ---------- - path : string - Location of database file. - buffers : bool, optional - If True (default) use support for buffers, which should increase performance by - reducing memory copies. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `lmdb.open` function. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.v2.LMDBStore('data/array.mdb') - >>> z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.v2.LMDBStore('data/group.mdb') - >>> root = zarr.v2.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - - After modifying a DBMStore, the ``close()`` method must be called, otherwise - essential data may not be written to the underlying database file. The - DBMStore class also supports the context manager protocol, which ensures the - ``close()`` method is called on leaving the context, e.g.:: - - >>> with zarr.v2.LMDBStore('data/array.mdb') as store: - ... z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - ... z[...] = 42 - ... # no need to call store.close() - - Notes - ----- - By default writes are not immediately flushed to disk to increase performance. You - can ensure data are flushed to disk by calling the ``flush()`` or ``close()`` methods. - - Should be safe to write in multiple threads or processes due to the synchronization - support within LMDB, although writing from multiple processes has not been tested. - - """ - - def __init__(self, path, buffers=True, dimension_separator=None, **kwargs): - import lmdb - - # set default memory map size to something larger than the lmdb default, which is - # very likely to be too small for any moderate array (logic copied from zict) - map_size = 2**40 if sys.maxsize >= 2**32 else 2**28 - kwargs.setdefault("map_size", map_size) - - # don't initialize buffers to zero by default, shouldn't be necessary - kwargs.setdefault("meminit", False) - - # decide whether to use the writemap option based on the operating system's - # support for sparse files - writemap requires sparse file support otherwise - # the whole# `map_size` may be reserved up front on disk (logic copied from zict) - writemap = sys.platform.startswith("linux") - kwargs.setdefault("writemap", writemap) - - # decide options for when data are flushed to disk - choose to delay syncing - # data to filesystem, otherwise pay a large performance penalty (zict also does - # this) - kwargs.setdefault("metasync", False) - kwargs.setdefault("sync", False) - kwargs.setdefault("map_async", False) - - # set default option for number of cached transactions - max_spare_txns = multiprocessing.cpu_count() - kwargs.setdefault("max_spare_txns", max_spare_txns) - - # normalize path - path = os.path.abspath(path) - - # open database - self.db = lmdb.open(path, **kwargs) - - # store properties - self.buffers = buffers - self.path = path - self.kwargs = kwargs - self._dimension_separator = dimension_separator - - def __getstate__(self): - try: - self.flush() # just in case - except Exception: - # flush may fail if db has already been closed - pass - return self.path, self.buffers, self.kwargs - - def __setstate__(self, state): - path, buffers, kwargs = state - self.__init__(path=path, buffers=buffers, **kwargs) - - def close(self): - """Closes the underlying database.""" - self.db.close() - - def flush(self): - """Synchronizes data to the file system.""" - self.db.sync() - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - def __getitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - # use the buffers option, should avoid a memory copy - with self.db.begin(buffers=self.buffers) as txn: - value = txn.get(key) - if value is None: - raise KeyError(key) - return value - - def __setitem__(self, key, value): - if isinstance(key, str): - key = key.encode("ascii") - with self.db.begin(write=True, buffers=self.buffers) as txn: - txn.put(key, value) - - def __delitem__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - with self.db.begin(write=True) as txn: - if not txn.delete(key): - raise KeyError(key) - - def __contains__(self, key): - if isinstance(key, str): - key = key.encode("ascii") - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - return cursor.set_key(key) - - def items(self): - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - for k, v in cursor.iternext(keys=True, values=True): - yield ensure_text(k, "ascii"), v - - def keys(self): - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - for k in cursor.iternext(keys=True, values=False): - yield ensure_text(k, "ascii") - - def values(self): - with self.db.begin(buffers=self.buffers) as txn: - with txn.cursor() as cursor: - yield from cursor.iternext(keys=False, values=True) - - def __iter__(self): - return self.keys() - - def __len__(self): - return self.db.stat()["entries"] - - -class LRUStoreCache(Store): - """Storage class that implements a least-recently-used (LRU) cache layer over - some other store. Intended primarily for use with stores that can be slow to - access, e.g., remote stores that require network communication to store and - retrieve data. - - Parameters - ---------- - store : Store - The store containing the actual data to be cached. - max_size : int - The maximum size that the cache may grow to, in number of bytes. Provide `None` - if you would like the cache to have unlimited size. - - Examples - -------- - The example below wraps an S3 store with an LRU cache:: - - >>> import s3fs - >>> import zarr - >>> s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(region_name='eu-west-2')) - >>> store = s3fs.S3Map(root='zarr-demo/store', s3=s3, check=False) - >>> cache = zarr.v2.LRUStoreCache(store, max_size=2**28) - >>> root = zarr.v2.group(store=cache) # doctest: +REMOTE_DATA - >>> z = root['foo/bar/baz'] # doctest: +REMOTE_DATA - >>> from timeit import timeit - >>> # first data access is relatively slow, retrieved from store - ... timeit('print(z[:].tobytes())', number=1, globals=globals()) # doctest: +SKIP - b'Hello from the cloud!' - 0.1081731989979744 - >>> # second data access is faster, uses cache - ... timeit('print(z[:].tobytes())', number=1, globals=globals()) # doctest: +SKIP - b'Hello from the cloud!' - 0.0009490990014455747 - - """ - - def __init__(self, store: StoreLike, max_size: int): - self._store: BaseStore = BaseStore._ensure_store(store) - self._max_size = max_size - self._current_size = 0 - self._keys_cache = None - self._contains_cache: Dict[Any, Any] = {} - self._listdir_cache: Dict[Path, Any] = dict() - self._values_cache: Dict[Path, Any] = OrderedDict() - self._mutex = Lock() - self.hits = self.misses = 0 - - def __getstate__(self): - return ( - self._store, - self._max_size, - self._current_size, - self._keys_cache, - self._contains_cache, - self._listdir_cache, - self._values_cache, - self.hits, - self.misses, - ) - - def __setstate__(self, state): - ( - self._store, - self._max_size, - self._current_size, - self._keys_cache, - self._contains_cache, - self._listdir_cache, - self._values_cache, - self.hits, - self.misses, - ) = state - self._mutex = Lock() - - def __len__(self): - return len(self._keys()) - - def __iter__(self): - return self.keys() - - def __contains__(self, key): - with self._mutex: - if key not in self._contains_cache: - self._contains_cache[key] = key in self._store - return self._contains_cache[key] - - def clear(self): - self._store.clear() - self.invalidate() - - def keys(self): - with self._mutex: - return iter(self._keys()) - - def _keys(self): - if self._keys_cache is None: - self._keys_cache = list(self._store.keys()) - return self._keys_cache - - def listdir(self, path: Path = None): - with self._mutex: - try: - return self._listdir_cache[path] - except KeyError: - listing = listdir(self._store, path) - self._listdir_cache[path] = listing - return listing - - def getsize(self, path=None) -> int: - return getsize(self._store, path=path) - - def _pop_value(self): - # remove the first value from the cache, as this will be the least recently - # used value - _, v = self._values_cache.popitem(last=False) - return v - - def _accommodate_value(self, value_size): - if self._max_size is None: - return - # ensure there is enough space in the cache for a new value - while self._current_size + value_size > self._max_size: - v = self._pop_value() - self._current_size -= buffer_size(v) - - def _cache_value(self, key: Path, value): - # cache a value - value_size = buffer_size(value) - # check size of the value against max size, as if the value itself exceeds max - # size then we are never going to cache it - if self._max_size is None or value_size <= self._max_size: - self._accommodate_value(value_size) - self._values_cache[key] = value - self._current_size += value_size - - def invalidate(self): - """Completely clear the cache.""" - with self._mutex: - self._values_cache.clear() - self._invalidate_keys() - self._current_size = 0 - - def invalidate_values(self): - """Clear the values cache.""" - with self._mutex: - self._values_cache.clear() - - def invalidate_keys(self): - """Clear the keys cache.""" - with self._mutex: - self._invalidate_keys() - - def _invalidate_keys(self): - self._keys_cache = None - self._contains_cache.clear() - self._listdir_cache.clear() - - def _invalidate_value(self, key): - if key in self._values_cache: - value = self._values_cache.pop(key) - self._current_size -= buffer_size(value) - - def __getitem__(self, key): - try: - # first try to obtain the value from the cache - with self._mutex: - value = self._values_cache[key] - # cache hit if no KeyError is raised - self.hits += 1 - # treat the end as most recently used - self._values_cache.move_to_end(key) - - except KeyError: - # cache miss, retrieve value from the store - value = self._store[key] - with self._mutex: - self.misses += 1 - # need to check if key is not in the cache, as it may have been cached - # while we were retrieving the value from the store - if key not in self._values_cache: - self._cache_value(key, value) - - return value - - def __setitem__(self, key, value): - self._store[key] = value - with self._mutex: - self._invalidate_keys() - self._invalidate_value(key) - self._cache_value(key, value) - - def __delitem__(self, key): - del self._store[key] - with self._mutex: - self._invalidate_keys() - self._invalidate_value(key) - - -class SQLiteStore(Store): - """Storage class using SQLite. - - Parameters - ---------- - path : string - Location of database file. - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `sqlite3.connect` function. - - Examples - -------- - Store a single array:: - - >>> import zarr - >>> store = zarr.v2.SQLiteStore('data/array.sqldb') - >>> z = zarr.v2.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) - >>> z[...] = 42 - >>> store.close() # don't forget to call this when you're done - - Store a group:: - - >>> store = zarr.v2.SQLiteStore('data/group.sqldb') - >>> root = zarr.v2.group(store=store, overwrite=True) - >>> foo = root.create_group('foo') - >>> bar = foo.zeros('bar', shape=(10, 10), chunks=(5, 5)) - >>> bar[...] = 42 - >>> store.close() # don't forget to call this when you're done - """ - - def __init__(self, path, dimension_separator=None, **kwargs): - import sqlite3 - - self._dimension_separator = dimension_separator - - # normalize path - if path != ":memory:": - path = os.path.abspath(path) - - # store properties - self.path = path - self.kwargs = kwargs - - # allow threading if SQLite connections are thread-safe - # - # ref: https://www.sqlite.org/releaselog/3_3_1.html - # ref: https://github.com/python/cpython/issues/71377 - check_same_thread = True - if sqlite3.sqlite_version_info >= (3, 3, 1): - check_same_thread = False - - # keep a lock for serializing mutable operations - self.lock = Lock() - - # open database - self.db = sqlite3.connect( - self.path, - detect_types=0, - isolation_level=None, - check_same_thread=check_same_thread, - **self.kwargs, - ) - - # handle keys as `str`s - self.db.text_factory = str - - # get a cursor to read/write to the database - self.cursor = self.db.cursor() - - # initialize database with our table if missing - with self.lock: - self.cursor.execute("CREATE TABLE IF NOT EXISTS zarr(k TEXT PRIMARY KEY, v BLOB)") - - def __getstate__(self): - if self.path == ":memory:": - raise PicklingError("Cannot pickle in-memory SQLite databases") - return self.path, self.kwargs - - def __setstate__(self, state): - path, kwargs = state - self.__init__(path=path, **kwargs) - - def close(self): - """Closes the underlying database.""" - - # close cursor and db objects - self.cursor.close() - self.db.close() - - def __getitem__(self, key): - value = self.cursor.execute("SELECT v FROM zarr WHERE (k = ?)", (key,)) - for (v,) in value: - return v - raise KeyError(key) - - def __setitem__(self, key, value): - self.update({key: value}) - - def __delitem__(self, key): - with self.lock: - self.cursor.execute("DELETE FROM zarr WHERE (k = ?)", (key,)) - if self.cursor.rowcount < 1: - raise KeyError(key) - - def __contains__(self, key): - cs = self.cursor.execute("SELECT COUNT(*) FROM zarr WHERE (k = ?)", (key,)) - for (has,) in cs: - has = bool(has) - return has - - def items(self): - kvs = self.cursor.execute("SELECT k, v FROM zarr") - yield from kvs - - def keys(self): - ks = self.cursor.execute("SELECT k FROM zarr") - for (k,) in ks: - yield k - - def values(self): - vs = self.cursor.execute("SELECT v FROM zarr") - for (v,) in vs: - yield v - - def __iter__(self): - return self.keys() - - def __len__(self): - cs = self.cursor.execute("SELECT COUNT(*) FROM zarr") - for (c,) in cs: - return c - - def update(self, *args, **kwargs): - args += (kwargs,) - - kv_list = [] - for dct in args: - for k, v in dct.items(): - v = ensure_contiguous_ndarray_like(v) - - # Accumulate key-value pairs for storage - kv_list.append((k, v)) - - with self.lock: - self.cursor.executemany("REPLACE INTO zarr VALUES (?, ?)", kv_list) - - def listdir(self, path=None): - path = normalize_storage_path(path) - sep = "_" if path == "" else "/" - keys = self.cursor.execute( - """ - SELECT DISTINCT SUBSTR(m, 0, INSTR(m, "/")) AS l FROM ( - SELECT LTRIM(SUBSTR(k, LENGTH(?) + 1), "/") || "/" AS m - FROM zarr WHERE k LIKE (? || "{sep}%") - ) ORDER BY l ASC - """.format(sep=sep), - (path, path), - ) - keys = list(map(operator.itemgetter(0), keys)) - return keys - - def getsize(self, path=None): - path = normalize_storage_path(path) - size = self.cursor.execute( - """ - SELECT COALESCE(SUM(LENGTH(v)), 0) FROM zarr - WHERE k LIKE (? || "%") AND - 0 == INSTR(LTRIM(SUBSTR(k, LENGTH(?) + 1), "/"), "/") - """, - (path, path), - ) - for (s,) in size: - return s - - def rmdir(self, path=None): - path = normalize_storage_path(path) - if path: - with self.lock: - self.cursor.execute('DELETE FROM zarr WHERE k LIKE (? || "/%")', (path,)) - else: - self.clear() - - def clear(self): - with self.lock: - self.cursor.executescript( - """ - BEGIN TRANSACTION; - DROP TABLE zarr; - CREATE TABLE zarr(k TEXT PRIMARY KEY, v BLOB); - COMMIT TRANSACTION; - """ - ) - - -class MongoDBStore(Store): - """Storage class using MongoDB. - - .. note:: This is an experimental feature. - - Requires the `pymongo `_ - package to be installed. - - Parameters - ---------- - database : string - Name of database - collection : string - Name of collection - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `pymongo.MongoClient` function. - - Notes - ----- - The maximum chunksize in MongoDB documents is 16 MB. - - """ - - _key = "key" - _value = "value" - - def __init__( - self, - database="mongodb_zarr", - collection="zarr_collection", - dimension_separator=None, - **kwargs, - ): - import pymongo - - self._database = database - self._collection = collection - self._dimension_separator = dimension_separator - self._kwargs = kwargs - - self.client = pymongo.MongoClient(**self._kwargs) - self.db = self.client.get_database(self._database) - self.collection = self.db.get_collection(self._collection) - - def __getitem__(self, key): - doc = self.collection.find_one({self._key: key}) - - if doc is None: - raise KeyError(key) - else: - return doc[self._value] - - def __setitem__(self, key, value): - value = ensure_bytes(value) - self.collection.replace_one( - {self._key: key}, {self._key: key, self._value: value}, upsert=True - ) - - def __delitem__(self, key): - result = self.collection.delete_many({self._key: key}) - if not result.deleted_count == 1: - raise KeyError(key) - - def __iter__(self): - for f in self.collection.find({}): - yield f[self._key] - - def __len__(self): - return self.collection.count_documents({}) - - def __getstate__(self): - return self._database, self._collection, self._kwargs - - def __setstate__(self, state): - database, collection, kwargs = state - self.__init__(database=database, collection=collection, **kwargs) - - def close(self): - """Cleanup client resources and disconnect from MongoDB.""" - self.client.close() - - def clear(self): - """Remove all items from store.""" - self.collection.delete_many({}) - - -class RedisStore(Store): - """Storage class using Redis. - - .. note:: This is an experimental feature. - - Requires the `redis `_ - package to be installed. - - Parameters - ---------- - prefix : string - Name of prefix for Redis keys - dimension_separator : {'.', '/'}, optional - Separator placed between the dimensions of a chunk. - **kwargs - Keyword arguments passed through to the `redis.Redis` function. - - """ - - def __init__(self, prefix="zarr", dimension_separator=None, **kwargs): - import redis - - self._prefix = prefix - self._kwargs = kwargs - self._dimension_separator = dimension_separator - - self.client = redis.Redis(**kwargs) - - def _key(self, key): - return "{prefix}:{key}".format(prefix=self._prefix, key=key) - - def __getitem__(self, key): - return self.client[self._key(key)] - - def __setitem__(self, key, value): - value = ensure_bytes(value) - self.client[self._key(key)] = value - - def __delitem__(self, key): - count = self.client.delete(self._key(key)) - if not count: - raise KeyError(key) - - def keylist(self): - offset = len(self._key("")) # length of prefix - return [key[offset:].decode("utf-8") for key in self.client.keys(self._key("*"))] - - def keys(self): - yield from self.keylist() - - def __iter__(self): - yield from self.keys() - - def __len__(self): - return len(self.keylist()) - - def __getstate__(self): - return self._prefix, self._kwargs - - def __setstate__(self, state): - prefix, kwargs = state - self.__init__(prefix=prefix, **kwargs) - - def clear(self): - for key in self.keys(): - del self[key] - - -class ConsolidatedMetadataStore(Store): - """A layer over other storage, where the metadata has been consolidated into - a single key. - - The purpose of this class, is to be able to get all of the metadata for - a given array in a single read operation from the underlying storage. - See :func:`zarr.v2.convenience.consolidate_metadata` for how to create this - single metadata key. - - This class loads from the one key, and stores the data in a dict, so that - accessing the keys no longer requires operations on the backend store. - - This class is read-only, and attempts to change the array metadata will - fail, but changing the data is possible. If the backend storage is changed - directly, then the metadata stored here could become obsolete, and - :func:`zarr.v2.convenience.consolidate_metadata` should be called again and the class - re-invoked. The use case is for write once, read many times. - - .. versionadded:: 2.3 - - .. note:: This is an experimental feature. - - Parameters - ---------- - store: Store - Containing the zarr array. - metadata_key: str - The target in the store where all of the metadata are stored. We - assume JSON encoding. - - See Also - -------- - zarr.v2.convenience.consolidate_metadata, zarr.v2.convenience.open_consolidated - - """ - - def __init__(self, store: StoreLike, metadata_key=".zmetadata"): - self.store = Store._ensure_store(store) - - # retrieve consolidated metadata - meta = json_loads(self.store[metadata_key]) - - # check format of consolidated metadata - consolidated_format = meta.get("zarr_consolidated_format", None) - if consolidated_format != 1: - raise MetadataError( - "unsupported zarr consolidated metadata format: %s" % consolidated_format - ) - - # decode metadata - self.meta_store: Store = KVStore(meta["metadata"]) - - def __getitem__(self, key): - return self.meta_store[key] - - def __contains__(self, item): - return item in self.meta_store - - def __iter__(self): - return iter(self.meta_store) - - def __len__(self): - return len(self.meta_store) - - def __delitem__(self, key): - raise ReadOnlyError - - def __setitem__(self, key, value): - raise ReadOnlyError - - def getsize(self, path): - return getsize(self.meta_store, path) - - def listdir(self, path): - return listdir(self.meta_store, path) diff --git a/src/zarr/v2/sync.py b/src/zarr/v2/sync.py deleted file mode 100644 index 49684a51ee..0000000000 --- a/src/zarr/v2/sync.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -from collections import defaultdict -from threading import Lock - -import fasteners - - -class ThreadSynchronizer: - """Provides synchronization using thread locks.""" - - def __init__(self): - self.mutex = Lock() - self.locks = defaultdict(Lock) - - def __getitem__(self, item): - with self.mutex: - return self.locks[item] - - def __getstate__(self): - return True - - def __setstate__(self, *args): - # reinitialize from scratch - self.__init__() - - -class ProcessSynchronizer: - """Provides synchronization using file locks via the - `fasteners `_ - package. - - Parameters - ---------- - path : string - Path to a directory on a file system that is shared by all processes. - N.B., this should be a *different* path to where you store the array. - - """ - - def __init__(self, path): - self.path = path - - def __getitem__(self, item): - path = os.path.join(self.path, item) - lock = fasteners.InterProcessLock(path) - return lock - - # pickling and unpickling should be handled automatically diff --git a/src/zarr/v2/util.py b/src/zarr/v2/util.py deleted file mode 100644 index 7e3bd788ec..0000000000 --- a/src/zarr/v2/util.py +++ /dev/null @@ -1,788 +0,0 @@ -import inspect -import json -import math -import numbers -from textwrap import TextWrapper -import mmap -import time -from typing import ( - Any, - Callable, - Dict, - Iterator, - Mapping, - Optional, - Tuple, - TypeVar, - Union, - Iterable, - cast, -) - -import numpy as np -import numpy.typing as npt -from asciitree import BoxStyle, LeftAligned -from asciitree.traversal import Traversal -from numcodecs.compat import ( - ensure_text, - ensure_ndarray_like, - ensure_bytes, - ensure_contiguous_ndarray_like, -) -from numcodecs.ndarray_like import NDArrayLike -from numcodecs.registry import codec_registry -from numcodecs.blosc import cbuffer_sizes, cbuffer_metainfo - -KeyType = TypeVar("KeyType") -ValueType = TypeVar("ValueType") - - -def flatten(arg: Iterable[Any]) -> Iterable[Any]: - for element in arg: - if isinstance(element, Iterable) and not isinstance(element, (str, bytes)): - yield from flatten(element) - else: - yield element - - -# codecs to use for object dtype convenience API -object_codecs = { - str.__name__: "vlen-utf8", - bytes.__name__: "vlen-bytes", - "array": "vlen-array", -} - - -class NumberEncoder(json.JSONEncoder): - def default(self, o): - # See json.JSONEncoder.default docstring for explanation - # This is necessary to encode numpy dtype - if isinstance(o, numbers.Integral): - return int(o) - if isinstance(o, numbers.Real): - return float(o) - return json.JSONEncoder.default(self, o) - - -def json_dumps(o: Any) -> bytes: - """Write JSON in a consistent, human-readable way.""" - return json.dumps( - o, indent=4, sort_keys=True, ensure_ascii=True, separators=(",", ": "), cls=NumberEncoder - ).encode("ascii") - - -def json_loads(s: Union[bytes, str]) -> Dict[str, Any]: - """Read JSON in a consistent way.""" - return json.loads(ensure_text(s, "utf-8")) - - -def normalize_shape(shape: Union[int, Tuple[int, ...], None]) -> Tuple[int, ...]: - """Convenience function to normalize the `shape` argument.""" - - if shape is None: - raise TypeError("shape is None") - - # handle 1D convenience form - if isinstance(shape, numbers.Integral): - shape = (int(shape),) - - # normalize - shape = cast(Tuple[int, ...], shape) - shape = tuple(int(s) for s in shape) - return shape - - -# code to guess chunk shape, adapted from h5py - -CHUNK_BASE = 256 * 1024 # Multiplier by which chunks are adjusted -CHUNK_MIN = 128 * 1024 # Soft lower limit (128k) -CHUNK_MAX = 64 * 1024 * 1024 # Hard upper limit - - -def guess_chunks(shape: Tuple[int, ...], typesize: int) -> Tuple[int, ...]: - """ - Guess an appropriate chunk layout for an array, given its shape and - the size of each element in bytes. Will allocate chunks only as large - as MAX_SIZE. Chunks are generally close to some power-of-2 fraction of - each axis, slightly favoring bigger values for the last index. - Undocumented and subject to change without warning. - """ - - ndims = len(shape) - # require chunks to have non-zero length for all dimensions - chunks = np.maximum(np.array(shape, dtype="=f8"), 1) - - # Determine the optimal chunk size in bytes using a PyTables expression. - # This is kept as a float. - dset_size = np.prod(chunks) * typesize - target_size = CHUNK_BASE * (2 ** np.log10(dset_size / (1024.0 * 1024))) - - if target_size > CHUNK_MAX: - target_size = CHUNK_MAX - elif target_size < CHUNK_MIN: - target_size = CHUNK_MIN - - idx = 0 - while True: - # Repeatedly loop over the axes, dividing them by 2. Stop when: - # 1a. We're smaller than the target chunk size, OR - # 1b. We're within 50% of the target chunk size, AND - # 2. The chunk is smaller than the maximum chunk size - - chunk_bytes = np.prod(chunks) * typesize - - if ( - chunk_bytes < target_size or abs(chunk_bytes - target_size) / target_size < 0.5 - ) and chunk_bytes < CHUNK_MAX: - break - - if np.prod(chunks) == 1: - break # Element size larger than CHUNK_MAX - - chunks[idx % ndims] = math.ceil(chunks[idx % ndims] / 2.0) - idx += 1 - - return tuple(int(x) for x in chunks) - - -def normalize_chunks(chunks: Any, shape: Tuple[int, ...], typesize: int) -> Tuple[int, ...]: - """Convenience function to normalize the `chunks` argument for an array - with the given `shape`.""" - - # N.B., expect shape already normalized - - # handle auto-chunking - if chunks is None or chunks is True: - return guess_chunks(shape, typesize) - - # handle no chunking - if chunks is False: - return shape - - # handle 1D convenience form - if isinstance(chunks, numbers.Integral): - chunks = tuple(int(chunks) for _ in shape) - - # handle bad dimensionality - if len(chunks) > len(shape): - raise ValueError("too many dimensions in chunks") - - # handle underspecified chunks - if len(chunks) < len(shape): - # assume chunks across remaining dimensions - chunks += shape[len(chunks) :] - - # handle None or -1 in chunks - if -1 in chunks or None in chunks: - chunks = tuple(s if c == -1 or c is None else int(c) for s, c in zip(shape, chunks)) - - chunks = tuple(int(c) for c in chunks) - return chunks - - -def normalize_dtype(dtype: Union[str, npt.DTypeLike], object_codec) -> Tuple[np.dtype[Any], Any]: - # convenience API for object arrays - if inspect.isclass(dtype): - dtype = dtype.__name__ - if isinstance(dtype, str): - # allow ':' to delimit class from codec arguments - tokens = dtype.split(":") - key = tokens[0] - if key in object_codecs: - dtype = np.dtype(object) - if object_codec is None: - codec_id = object_codecs[key] - if len(tokens) > 1: - args = tokens[1].split(",") - else: - args = [] - try: - object_codec = codec_registry[codec_id](*args) - except KeyError: # pragma: no cover - raise ValueError( - "codec %r for object type %r is not " - "available; please provide an " - "object_codec manually" % (codec_id, key) - ) - return dtype, object_codec - - dtype = np.dtype(dtype) - - # don't allow generic datetime64 or timedelta64, require units to be specified - if dtype == np.dtype("M8") or dtype == np.dtype("m8"): - raise ValueError( - "datetime64 and timedelta64 dtypes with generic units " - 'are not supported, please specify units (e.g., "M8[ns]")' - ) - - return dtype, object_codec - - -# noinspection PyTypeChecker -def is_total_slice(item, shape: Tuple[int]) -> bool: - """Determine whether `item` specifies a complete slice of array with the - given `shape`. Used to optimize __setitem__ operations on the Chunk - class.""" - - # N.B., assume shape is normalized - - if item == Ellipsis: - return True - if item == slice(None): - return True - if isinstance(item, slice): - item = (item,) - if isinstance(item, tuple): - return all( - ( - isinstance(it, slice) - and ((it == slice(None)) or ((it.stop - it.start == sh) and (it.step in [1, None]))) - ) - for it, sh in zip(item, shape) - ) - else: - raise TypeError("expected slice or tuple of slices, found %r" % item) - - -def normalize_resize_args(old_shape, *args): - # normalize new shape argument - if len(args) == 1: - new_shape = args[0] - else: - new_shape = args - if isinstance(new_shape, int): - new_shape = (new_shape,) - else: - new_shape = tuple(new_shape) - if len(new_shape) != len(old_shape): - raise ValueError("new shape must have same number of dimensions") - - # handle None in new_shape - new_shape = tuple(s if n is None else int(n) for s, n in zip(old_shape, new_shape)) - - return new_shape - - -def human_readable_size(size) -> str: - if size < 2**10: - return "%s" % size - elif size < 2**20: - return "%.1fK" % (size / float(2**10)) - elif size < 2**30: - return "%.1fM" % (size / float(2**20)) - elif size < 2**40: - return "%.1fG" % (size / float(2**30)) - elif size < 2**50: - return "%.1fT" % (size / float(2**40)) - else: - return "%.1fP" % (size / float(2**50)) - - -def normalize_order(order: str) -> str: - order = str(order).upper() - if order not in ["C", "F"]: - raise ValueError("order must be either 'C' or 'F', found: %r" % order) - return order - - -def normalize_dimension_separator(sep: Optional[str]) -> Optional[str]: - if sep in (".", "/", None): - return sep - else: - raise ValueError("dimension_separator must be either '.' or '/', found: %r" % sep) - - -def normalize_fill_value(fill_value, dtype: np.dtype[Any]): - if fill_value is None or dtype.hasobject: - # no fill value - pass - elif not isinstance(fill_value, np.void) and fill_value == 0: - # this should be compatible across numpy versions for any array type, including - # structured arrays - fill_value = np.zeros((), dtype=dtype)[()] - - elif dtype.kind == "U": - # special case unicode because of encoding issues on Windows if passed through numpy - # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713 - - if not isinstance(fill_value, str): - raise ValueError( - "fill_value {!r} is not valid for dtype {}; must be a unicode string".format( - fill_value, dtype - ) - ) - - else: - try: - if isinstance(fill_value, bytes) and dtype.kind == "V": - # special case for numpy 1.14 compatibility - fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()] - else: - fill_value = np.array(fill_value, dtype=dtype)[()] - - except Exception as e: - # re-raise with our own error message to be helpful - raise ValueError( - "fill_value {!r} is not valid for dtype {}; nested exception: {}".format( - fill_value, dtype, e - ) - ) - - return fill_value - - -def normalize_storage_path(path: Union[str, bytes, None]) -> str: - # handle bytes - if isinstance(path, bytes): - path = str(path, "ascii") - - # ensure str - if path is not None and not isinstance(path, str): - path = str(path) - - if path: - # convert backslash to forward slash - path = path.replace("\\", "/") - - # ensure no leading slash - while len(path) > 0 and path[0] == "/": - path = path[1:] - - # ensure no trailing slash - while len(path) > 0 and path[-1] == "/": - path = path[:-1] - - # collapse any repeated slashes - previous_char = None - collapsed = "" - for char in path: - if char == "/" and previous_char == "/": - pass - else: - collapsed += char - previous_char = char - path = collapsed - - # don't allow path segments with just '.' or '..' - segments = path.split("/") - if any(s in {".", ".."} for s in segments): - raise ValueError("path containing '.' or '..' segment not allowed") - - else: - path = "" - - return path - - -def buffer_size(v) -> int: - return ensure_ndarray_like(v).nbytes - - -def info_text_report(items: Dict[Any, Any]) -> str: - keys = [k for k, v in items] - max_key_len = max(len(k) for k in keys) - report = "" - for k, v in items: - wrapper = TextWrapper( - width=80, - initial_indent=k.ljust(max_key_len) + " : ", - subsequent_indent=" " * max_key_len + " : ", - ) - text = wrapper.fill(str(v)) - report += text + "\n" - return report - - -def info_html_report(items) -> str: - report = '' - report += "" - for k, v in items: - report += ( - "" - '' - '' - "" % (k, v) - ) - report += "" - report += "
%s%s
" - return report - - -class InfoReporter: - def __init__(self, obj): - self.obj = obj - - def __repr__(self): - items = self.obj.info_items() - return info_text_report(items) - - def _repr_html_(self): - items = self.obj.info_items() - return info_html_report(items) - - -class TreeNode: - def __init__(self, obj, depth=0, level=None): - self.obj = obj - self.depth = depth - self.level = level - - def get_children(self): - if hasattr(self.obj, "values") and (self.level is None or self.depth < self.level): - depth = self.depth + 1 - return [TreeNode(o, depth=depth, level=self.level) for o in self.obj.values()] - return [] - - def get_text(self): - name = self.obj.name.split("/")[-1] or "/" - if hasattr(self.obj, "shape"): - name += " {} {}".format(self.obj.shape, self.obj.dtype) - return name - - def get_type(self): - return type(self.obj).__name__ - - -class TreeTraversal(Traversal): # type: ignore[misc] - def get_children(self, node): - return node.get_children() - - def get_root(self, tree): - return tree - - def get_text(self, node): - return node.get_text() - - -tree_group_icon = "folder" -tree_array_icon = "table" - - -def tree_get_icon(stype: str) -> str: - if stype == "Array": - return tree_array_icon - elif stype == "Group": - return tree_group_icon - else: - raise ValueError("Unknown type: %s" % stype) - - -def tree_widget_sublist(node, root=False, expand=False): - import ipytree - - result = ipytree.Node() - result.icon = tree_get_icon(node.get_type()) - if root or (expand is True) or (isinstance(expand, int) and node.depth < expand): - result.opened = True - else: - result.opened = False - result.name = node.get_text() - result.nodes = [tree_widget_sublist(c, expand=expand) for c in node.get_children()] - result.disabled = True - - return result - - -def tree_widget(group, expand, level): - try: - import ipytree - except ImportError as error: - raise ImportError( - "{}: Run `pip install zarr[jupyter]` or `conda install ipytree`" - "to get the required ipytree dependency for displaying the tree " - "widget. If using jupyterlab<3, you also need to run " - "`jupyter labextension install ipytree`".format(error) - ) - - result = ipytree.Tree() - root = TreeNode(group, level=level) - result.add_node(tree_widget_sublist(root, root=True, expand=expand)) - - return result - - -class TreeViewer: - def __init__(self, group, expand=False, level=None): - self.group = group - self.expand = expand - self.level = level - - self.text_kwargs = dict(horiz_len=2, label_space=1, indent=1) - - self.bytes_kwargs = dict( - UP_AND_RIGHT="+", HORIZONTAL="-", VERTICAL="|", VERTICAL_AND_RIGHT="+" - ) - - self.unicode_kwargs = dict( - UP_AND_RIGHT="\u2514", - HORIZONTAL="\u2500", - VERTICAL="\u2502", - VERTICAL_AND_RIGHT="\u251c", - ) - - def __bytes__(self): - drawer = LeftAligned( - traverse=TreeTraversal(), draw=BoxStyle(gfx=self.bytes_kwargs, **self.text_kwargs) - ) - root = TreeNode(self.group, level=self.level) - result = drawer(root) - - # Unicode characters slip in on Python 3. - # So we need to straighten that out first. - result = result.encode() - - return result - - def __unicode__(self): - drawer = LeftAligned( - traverse=TreeTraversal(), draw=BoxStyle(gfx=self.unicode_kwargs, **self.text_kwargs) - ) - root = TreeNode(self.group, level=self.level) - return drawer(root) - - def __repr__(self): - return self.__unicode__() - - def _repr_mimebundle_(self, **kwargs): - tree = tree_widget(self.group, expand=self.expand, level=self.level) - return tree._repr_mimebundle_(**kwargs) - - -def check_array_shape(param, array, shape): - if not hasattr(array, "shape"): - raise TypeError( - "parameter {!r}: expected an array-like object, got {!r}".format(param, type(array)) - ) - if array.shape != shape: - raise ValueError( - "parameter {!r}: expected array with shape {!r}, got {!r}".format( - param, shape, array.shape - ) - ) - - -def is_valid_python_name(name): - from keyword import iskeyword - - return name.isidentifier() and not iskeyword(name) - - -class NoLock: - """A lock that doesn't lock.""" - - def __enter__(self): - pass - - def __exit__(self, *args): - pass - - -nolock = NoLock() - - -class PartialReadBuffer: - def __init__(self, store_key, chunk_store): - self.chunk_store = chunk_store - # is it fsstore or an actual fsspec map object - assert hasattr(self.chunk_store, "map") - self.map = self.chunk_store.map - self.fs = self.chunk_store.fs - self.store_key = store_key - self.buff = None - self.nblocks = None - self.start_points = None - self.n_per_block = None - self.start_points_max = None - self.read_blocks = set() - - _key_path = self.map._key_to_str(store_key) - _key_path = _key_path.split("/") - _chunk_path = [self.chunk_store._normalize_key(_key_path[-1])] - _key_path = "/".join(_key_path[:-1] + _chunk_path) - self.key_path = _key_path - - def prepare_chunk(self): - assert self.buff is None - header = self.fs.read_block(self.key_path, 0, 16) - nbytes, self.cbytes, blocksize = cbuffer_sizes(header) - typesize, _shuffle, _memcpyd = cbuffer_metainfo(header) - self.buff = mmap.mmap(-1, self.cbytes) - self.buff[0:16] = header - self.nblocks = nbytes / blocksize - self.nblocks = ( - int(self.nblocks) if self.nblocks == int(self.nblocks) else int(self.nblocks + 1) - ) - if self.nblocks == 1: - self.buff = self.read_full() - return - start_points_buffer = self.fs.read_block(self.key_path, 16, int(self.nblocks * 4)) - self.start_points = np.frombuffer(start_points_buffer, count=self.nblocks, dtype=np.int32) - self.start_points_max = self.start_points.max() - self.buff[16 : (16 + (self.nblocks * 4))] = start_points_buffer - self.n_per_block = blocksize / typesize - - def read_part(self, start, nitems): - assert self.buff is not None - if self.nblocks == 1: - return - start_block = int(start / self.n_per_block) - wanted_decompressed = 0 - while wanted_decompressed < nitems: - if start_block not in self.read_blocks: - start_byte = self.start_points[start_block] - if start_byte == self.start_points_max: - stop_byte = self.cbytes - else: - stop_byte = self.start_points[self.start_points > start_byte].min() - length = stop_byte - start_byte - data_buff = self.fs.read_block(self.key_path, start_byte, length) - self.buff[start_byte:stop_byte] = data_buff - self.read_blocks.add(start_block) - if wanted_decompressed == 0: - wanted_decompressed += ((start_block + 1) * self.n_per_block) - start - else: - wanted_decompressed += self.n_per_block - start_block += 1 - - def read_full(self): - return self.chunk_store[self.store_key] - - -class UncompressedPartialReadBufferV3: - def __init__(self, store_key, chunk_store, itemsize): - assert chunk_store.supports_efficient_get_partial_values - self.chunk_store = chunk_store - self.store_key = store_key - self.itemsize = itemsize - - def prepare_chunk(self): - pass - - def read_part(self, start, nitems): - return self.chunk_store.get_partial_values( - [(self.store_key, (start * self.itemsize, nitems * self.itemsize))] - )[0] - - def read_full(self): - return self.chunk_store[self.store_key] - - -def retry_call( - callabl: Callable[..., Any], - args=None, - kwargs=None, - exceptions: Tuple[Any, ...] = (), - retries: int = 10, - wait: float = 0.1, -) -> Any: - """ - Make several attempts to invoke the callable. If one of the given exceptions - is raised, wait the given period of time and retry up to the given number of - retries. - """ - - if args is None: - args = () - if kwargs is None: - kwargs = {} - - for attempt in range(1, retries + 1): - try: - return callabl(*args, **kwargs) - except exceptions: - if attempt < retries: - time.sleep(wait) - else: - raise - - -def all_equal(value: Any, array: Any): - """ - Test if all the elements of an array are equivalent to a value. - If `value` is None, then this function does not do any comparison and - returns False. - """ - - if value is None: - return False - if not value: - # if `value` is falsey, then just 1 truthy value in `array` - # is sufficient to return False. We assume here that np.any is - # optimized to return on the first truthy value in `array`. - try: - return not np.any(array) - except (TypeError, ValueError): # pragma: no cover - pass - if np.issubdtype(array.dtype, np.object_): - # we have to flatten the result of np.equal to handle outputs like - # [np.array([True,True]), True, True] - return all(flatten(np.equal(value, array, dtype=array.dtype))) - else: - # Numpy errors if you call np.isnan on custom dtypes, so ensure - # we are working with floats before calling isnan - if np.issubdtype(array.dtype, np.floating) and np.isnan(value): - return np.all(np.isnan(array)) - else: - # using == raises warnings from numpy deprecated pattern, but - # using np.equal() raises type errors for structured dtypes... - return np.all(value == array) - - -def ensure_contiguous_ndarray_or_bytes(buf) -> Union[NDArrayLike, bytes]: - """Convenience function to coerce `buf` to ndarray-like array or bytes. - - First check if `buf` can be zero-copy converted to a contiguous array. - If not, `buf` will be copied to a newly allocated `bytes` object. - - Parameters - ---------- - buf : ndarray-like, array-like, or bytes-like - A numpy array like object such as numpy.ndarray, cupy.ndarray, or - any object exporting a buffer interface. - - Returns - ------- - arr : NDArrayLike or bytes - A ndarray-like or bytes object - """ - - try: - return ensure_contiguous_ndarray_like(buf) - except TypeError: - # An error is raised if `buf` couldn't be zero-copy converted - return ensure_bytes(buf) - - -class ConstantMap(Mapping[KeyType, ValueType]): - """A read-only map that maps all keys to the same constant value - - Useful if you want to call `getitems()` with the same context for all keys. - - Parameters - ---------- - keys - The keys of the map. Will be copied to a frozenset if it isn't already. - constant - The constant that all keys are mapping to. - """ - - def __init__(self, keys: Iterable[KeyType], constant: ValueType) -> None: - self._keys = keys if isinstance(keys, frozenset) else frozenset(keys) - self._constant = constant - - def __getitem__(self, key: KeyType) -> ValueType: - if key not in self._keys: - raise KeyError(repr(key)) - return self._constant - - def __iter__(self) -> Iterator[KeyType]: - return iter(self._keys) - - def __len__(self) -> int: - return len(self._keys) - - def __contains__(self, key: object) -> bool: - return key in self._keys - - def __repr__(self) -> str: - return repr(dict(self.items())) diff --git a/tests/v2/__init__.py b/tests/v2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/v2/conftest.py b/tests/v2/conftest.py deleted file mode 100644 index 6680e4066b..0000000000 --- a/tests/v2/conftest.py +++ /dev/null @@ -1,7 +0,0 @@ -import pytest -import pathlib - - -@pytest.fixture(params=[str, pathlib.Path]) -def path_type(request): - return request.param diff --git a/tests/v2/fixture/.zgroup b/tests/v2/fixture/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/tests/v2/fixture/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/dimension_separator/flat/.zarray b/tests/v2/fixture/dimension_separator/flat/.zarray deleted file mode 100644 index f265bb0674..0000000000 --- a/tests/v2/fixture/dimension_separator/flat/.zarray +++ /dev/null @@ -1,23 +0,0 @@ -{ - "chunks": [ - 2, - 2 - ], - "compressor": { - "blocksize": 0, - "clevel": 5, - "cname": "lz4", - "id": "blosc", - "shuffle": 1 - }, - "dimension_separator": ".", - "dtype": "MC+6cQE@6%&_`l#-T_m6KOcR8m$^Ra4i{)Y8_`)zddH zG%_|ZH8Z!cw6eCbwX=6{baHlab#wRd^z!!c_45x13RUz zF>}`JIdkXDU$Ah|;w4L$Enl&6)#^2C*R9{Mant54TeofBv2)k%J$v`RUzF>}`JIdkXDU$Ah|;w4L$Enl&6)#^2C*R9{Mant54TeofB zv2)k%J$v`+{(>E|QGBzQN=`{lOV7y6%FfBn%P%M_DlRE4E3c@ms;;T6t8Zv*YHn$5Ywzgn>h9_7>z^=j z(&Q;qr%j(RbJpxRbLY)puyE1hB}NRWEt>3V5)8;K(w{73CbJy-Yd-v@> zaPZLKBS()NKXLNZ=`&}~oxgDL(&Z~xuU)@!^VaPo;%T iz5np>)8{W=zkUDl^Vjb`fB*ew81+9*;vWD^rVa$L$DH;6 diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_1/1 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_1/1 deleted file mode 100644 index 4a92f77cede08276ecabe701f17af40e362bc6ba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 285 zcmV+&0pk960gH%?ijIkmi%&>QN=`{lOV7y6%FfBn%P%M_DlRE4E3c@ms;;T6t8Zv* zYHn$5Ywzgn>h9_7>z^=j(&Q;qr%j(RbJpxRbLY)puyE1hB}NRWEt>3V5 z)8;K(w{73CbJy-Yd-v@>aPZLKBS()NKXLNZ=`&}~oxgDL(&Z~xuU)@!^VaPo;%Tz5np>)8{W=zkUDl^Vjb`fB*ewU}R!uVP#|I;N;@w;pO8O z5EK#?5fu}ckd%^^k(HBIP*hS@QB_md(A3h_(bdy8Ff=kYF*P%{u(Yzav9+^zaCCBZ jadmU|@bvQb@%8f$2n-4i2@MO67}@@3pcMcBHWU7;>t&vN diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_2/.zarray b/tests/v2/fixture/test_format_compatibility/array_0/compressor_2/.zarray deleted file mode 100644 index 2b3043fb76..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_0/compressor_2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 600 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_2/0 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_2/0 deleted file mode 100644 index ae9adf56dfa9c9deb5a76e8319d282cc4115098e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 525 zcmZ>Y$}lu^j8qGb{NHNyh=GBz{y!nGfw7>0Yt^dE1_t~f{MD)+hm;GvMh1=x7dvxK z_Lbl;F=FOOF*zVC*x=ZpaKhxWvjAH|215qp0z)Y_HD|{OmmMxMFbO3v%wXWeXS)0o z^B9*#6)!#4GrPot-ozf+Xt0NK>65+Un|{Q5f2OyF|oNk=xW>DD^4c`dism91;L zv+itP$KCa0=eq7yZ+5TeKJ{hq`tDbM_HW?fW;wW_N1NyHMjmgGqZ@m&WsYy+=~g+p zsb{s$>CHT+P0nuad2Mrk3op0J#Vx(sK9{%hdWT%y+M6A7eH(9g%FS)Pt8;E|=RI9= zcYE*an)^HWxLY3X=+o|bypzv+%F~^F*>j%n;_F`Wa#!E#HLrK`o!;_xci-zh@AvR? zANjbaU;E7Gz5L!+zV7YMzVm$_fA^E0`}$YE`Msb2^q0T;`(OXjd(u^~Q1L%k>Jb2s CwC)Q4 diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_2/1 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_2/1 deleted file mode 100644 index edec761d33967bfb0f15a5b8d6f533d614d1ba65..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 500 zcmZ>Y$}lu^j8qGb%wCyrk%1xVKS8j8F@ZsA)v^W#2Bwq)K*FGhf!Top2sj%+Bn?wcBYqH8qr=Y2-nMqO8RbMtG%~Z==lr>xJ<)Nav>Y0zK=BvMA z{#BpF)v|C!|HpqTRvGmz(bNi>woG$X(YzH}T9cNoQV6wLwK}Z%>ebfNsIaR?OXI?? zv9=~fTyve87J038X;##=qg(T$ud^O4in;E3wJi2}>(i>Z>qo!-*FXK6;lKtTX^ul1 z&R7c^+2|83actw6YK0SKh$# zZR^?Ugd5v@XJ_2nc6N8co$bE2EADMS`@7-64nOIRM?210Pk6G^FM7tao#(0-yx8S8 zd&R3==XP&+v)k|Xj(5Ay{XXzvkH7SZPy4^md|o9tedX(Fx$8UM*T{>X{9G#^e)D^s V{PdSBf4g@o&C&2My|N*r0RTvL=cE7t diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_3/.zarray b/tests/v2/fixture/test_format_compatibility/array_0/compressor_3/.zarray deleted file mode 100644 index a5c1d0b86a..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_0/compressor_3/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 600 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_3/0 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_3/0 deleted file mode 100644 index ffaee5493bac4c138c00e1841cae757d68b59ec0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 300 zcmV+{0n`2h0gwS$0ssJ50ssIk0RR9L0000K0RR9fwJ-f(SOHZC00a;K0RjUA1qKHQ z2?`4g4Gs?w5fT#=6&4p585$cL9UdPbAtECrB_<~*DJm;0EiNxGF)}kWH8wXmIXXK$ zJw87`K|(`BMMg(RNlHshO-@fxQBqS>RaRG6Sz23MU0z>cVPa!sWoBn+X=-b1ZEkOH zadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^?m6n&7nVOrNot~edp`xRt zrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T$;!*j&Cbuz(bCh@)z;V8 y+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0({r>*}00313>Y4)cX@)TX diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_3/1 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_3/1 deleted file mode 100644 index 6ddcd64e7282440c30988f063fe6a05d9ed9863b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 303 zcmV+~0nq*e0gwS$0ssJ50ssIn0RR9L0000N0RR9fwJ-f(SOIMa02B~dSz23MU0z>c zVPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^? zm6n&7nVOrNot~edp`xRtrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T z$;!*j&Cbuz(bCh@)z;V8+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0( z{r>*|0RjUA1qKHQ2?`4g4Gs?w5fT#=6&4p585$cL9UdPbAtECrB_<~*DJm;0EiNxG zF)}kWH8wXmIXXK$Jw87`K|(`BMMg(RNlHshO-@fxQBqS>RaRF30sw_i0sw!2T2q*P BhXeos diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_4/.zarray b/tests/v2/fixture/test_format_compatibility/array_0/compressor_4/.zarray deleted file mode 100644 index 6d725e7208..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_0/compressor_4/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 600 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_4/0 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_4/0 deleted file mode 100644 index f54919df59f41ef78172b16bf1c0de116997ffb5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 300 zcmV+{0n`2h0g(Y%0ssJ50ssIk0RR9L0000K0RR9fwJ-f(SOHZC00a;K0RjUA1qKHQ z2?`4g4Gs?w5fT#=6&4p585$cL9UdPbAtECrB_<~*DJm;0EiNxGF)}kWH8wXmIXXK$ zJw87`K|(`BMMg(RNlHshO-@fxQBqS>RaRG6Sz23MU0z>cVPa!sWoBn+X=-b1ZEkOH zadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^?m6n&7nVOrNot~edp`xRt zrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T$;!*j&Cbuz(bCh@)z;V8 y+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0({r>*}00313>Y4)dlZG+? diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_4/1 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_4/1 deleted file mode 100644 index 9c4e54e9bbef65e016903a2bfb2ff44d87b25834..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 303 zcmV+~0nq*e0g(Y%0ssJ50ssIn0RR9L0000N0RR9fwJ-f(SOIMa02B~dSz23MU0z>c zVPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^? zm6n&7nVOrNot~edp`xRtrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T z$;!*j&Cbuz(bCh@)z;V8+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0( z{r>*|0RjUA1qKHQ2?`4g4Gs?w5fT#=6&4p585$cL9UdPbAtECrB_<~*DJm;0EiNxG zF)}kWH8wXmIXXK$Jw87`K|(`BMMg(RNlHshO-@fxQBqS>RaRF30sw_i0sw!2T2q;- BhXnut diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_5/.zarray b/tests/v2/fixture/test_format_compatibility/array_0/compressor_5/.zarray deleted file mode 100644 index e218eb4f93..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_0/compressor_5/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 600 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 2 - }, - "dtype": "|i1", - "fill_value": 0, - "filters": null, - "order": "F", - "shape": [ - 1111 - ], - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/test_format_compatibility/array_0/compressor_5/0 b/tests/v2/fixture/test_format_compatibility/array_0/compressor_5/0 deleted file mode 100644 index b72d1fa45bbb4594c693161fef240f1e3419cc53..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 121 zcmZQ#oWdBv#J~W;l?)6FB0!u7#2Q=m|0YB*?qy<7VOq87%m;@53=IE)7({{rPyhrW zqT&n-4mcchNb++JM}E!Fs)jp#c;-l;X?t#|NjgO|LcJi zDqvucV`w<$ki^HMC+6cQE@6%&_`l#-T_m6KOcR8m$^ zRa4i{)Y8_`)zddHG%_|ZH8Z!cw6eCbwX=6{baHlab#wRd^z!!c_45x13RUzF>}`JIdkXDU$Ah|;w4L$Enl&6)#^2C*R9{Mant54TeofBv2)k% zJ$v`@beFH-yV-r&| ya|=r=Ya3fTdk04+XBSsDcMnf5Zy#Sj|A4@t;E>R;@Dc8RU_yWZP6mqrFbe=kC-QWF#jADM>|Y(vX&Pq$dLz$wX$dkd00k*T zVTw?cViczYB`HN|%21Yal&1m}sYGR}P?c&_rv^2tMQ!R(mwMEv0S#$HW17&EW;CY- zEont-+R&DEw5J0d=|pF`@DN>jm`8Y&Zgi&ykMTH9(32<=Q&33JR^C5QM|}##xRy~jAsH9d5M>q#AK#0m1#_81~YkuSDD3ZUgLG< zFqe7EX8{X&gEv{kVwSL!w|JXnEN2BPS;cDBu$FbK=N;Z<0~>jd_xXTLY-S4|@)29v z#>Z^u6Fy}JJK4o!$G3dP zc`oohKky?Lxx{6z@Do4t3s?D-Yh33Bzj2dW+~#-gaF=`h!Jquaeg5Vj9`G-yVFDAH z$iyZwsmV-k3R9ZO)TS}5=}d11Gn&cFW-+VT%x(^In#jpRap6~mCo80UcKlCHFy3LQ>?k9fg z4tKiC-G1ih?s2bQxX=9_@JkPR$ip7-s9*WD$2{%{PkPF4Jnb3Jdd_eC&huXIdw=jp zFM7$#UhyY?_7|`EtJl2l4S(~dx4iA|-tn&Y{KG%}%lrQAKR)o^|MMFW2_j)6io}s5 zl18#f9w{Paq>9v$CelW_NFNy@V`PfVktMQ5w#XhiB4^}^+>s~pM!v`&1)^XSio#JO zibk<09wnk=l#0?(Cdx*+C?6G~VpNLCQ6;KIwWuC7qGr^J+EFL!M!l#X4WeN*ipJ3- fnntr|9xb9}w2IczCfY{3XdfM-V|0qn(Ix%^h3XHq diff --git a/tests/v2/fixture/test_format_compatibility/array_1/compressor_0/1 b/tests/v2/fixture/test_format_compatibility/array_1/compressor_0/1 deleted file mode 100644 index 7dd7d7404e17b12ad66cc591f7b729c816a18581..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1200 zcmdVH1A-U;00Y5mHnnZrwr$(CZQHhO+qP}nHs8^iYO1aps;OG4tvaf!daADmYN$qP ztR`xzW@@e$YN=Lgtu|__c51H<>ZnfYtS;)RZtAWc>ZxAptv>3je(J9Q8mK`UtRWhz zVH&Ox8mUnltuY#_aT>1)ny5*dtSOqRX_~GXnyFcutvQ;jd77^UTBt=@tR-5iWm>Kk zTB%i9tu+%AI;m4S ztus2Sb2_gJx~NOKtSh>zYr3u*x~W^btvkA_d%CX&dZtv7nB zcY3c6`lwI(tS|biZ~Cqu`l(;~tv~v!00JSv1SSYU2}W>25Ry=YCJbQ-M|dI-kw`=) z3Q>thbYc*bSi~j{afwHK5|EHYBqj+-Nk(!~kdjoSCJkvxM|v`lkxXPJ3t7oVc5;xD zT;wJXdC5n93Q&+j6s8D8DMoQhP?A!VrVM2%M|mnxkxEpi3RS5_b!t$PTGXZvb*V>v z8qknNG^PnnX-0Ee(2`cPrVVXrM|(QZkxq1`3tj0(cY4s1Ui799ed$Mk1~8C83}y&J z8OCr%Fp^P>W(;E)$9N_%kx5Ku3R9WJbY?J-EM^HyS;lf!u##1* zW({ju$9gufkxgu73tQR7c6P9nUF>ELd)dc+4seh|9Oei|ImU5LaFSD;<_u>!$9XPr zkxN|W3Rk(tb#8EzTioUjce%%X9`KMyJmv{cdB$^I@RC=&<_&Lo$9q2TkxzW)3t#!h zcYg4bU;O3|e+h6P%h(R=AP(wa4(<>R=}->sFb?Z*4(|w#=tz$2D30oAj_w$a=~#~K zIF9Rhj_(9c=tNHJBu?sNPVN*==~PbbG*0VuPVWrP=uFP+EY9j|&h8w}>0Hk3JkINU p&hG*)=t3^+A};D;F76U8=~6E3GA`?KF7FDi=t{2aDh~MX`~yF<1Z4mK diff --git a/tests/v2/fixture/test_format_compatibility/array_1/compressor_1/.zarray b/tests/v2/fixture/test_format_compatibility/array_1/compressor_1/.zarray deleted file mode 100644 index e4e48baf94..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_1/compressor_1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 600 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "+=(3WiI9o}UF8+niS`G8Gq zW(yzk5nI{D$86^lK4k|x*~M-?<8$_~moM1Ieh%;@2RX!Hj&PK(_?lxJ=L9D?#W$Sh z3}-pVw|vKWF7Q1+@FN$w#AUAV6F>6{SNWA|T;~SAag$rz=6CLJmwWudpZvvr{^lPZ z@Gq!g0u!3Z#3nJR$xLnvQ<}=urZKJQOm7A=n#s&&F{|0kZVq#r%iQKMuldYx0Sj8l z!WOZp#Vl?KOIpg(ma(kmEN=xXTFJ^*v8vUqZVhW%%i7kluJx>M0~^}N#x}93&1`N9 zTiVLjwy~}4Y;OlU+R4s#@gckVu#fnt-Ry1;AMoNh1~>Yi@B4w9-0T)V^dq;r z&5zygCw}SQ77s~y{I1z qqG2?O#?d62Mzd%hEuv+#iq_F4+D5x*A047&bc)W=CH@12>JPL*h}41r diff --git a/tests/v2/fixture/test_format_compatibility/array_1/compressor_1/1 b/tests/v2/fixture/test_format_compatibility/array_1/compressor_1/1 deleted file mode 100644 index 193648d77f656fd559439ae77084dc4b3c9226c2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 770 zcmYkqSxnOa002-F)SVsT1r-$)t9aCTq11zLwTz>T+m%bMDn)eMtQ;oB9}=^4n?Edm znx%L7P!ktFBr#ivX2C=gnI`_2W$P|J{LsY@e|>p#+rF%2KhDp}%3d2SeKq#&^?yr$ zd|IJ+8RR*Fmk?f{d1WWXqLc_xJW0v>DXxQ(N-1H6QUI03RS{PC2daEn<$6`=mMSc% z$~KU#0L7yq-w(>WL9Q8;E`Y)~sA#>c#VeM0`4O*N;N@&ysmv=lyvh!Ots%q{1V2p3 z`v|U`kggEIG@%&7Y-3nF7v{&p@}V%-6P9j>#?$>1GC9_e_s|b`G7UtRCXC518h#MR0>bFZ;BOBj_Zt;qG|pGmed5^T>t^pYrK@ zV(2Wr;pzS5GcCrjM?1|-g@M#m6$I?4+1>npt>7K7c0Xqy~?$=)q@=i7+!p(8kd zYs=NN5YhMj3Cu2SdHViFOpmnoq7PqzL=m9g0aCd@%nBsUK->Y$}lu^j8qGbd{UyW&%nS?|DO=Jz-*y#j-T;K15hyK zkWtYwv2o$SM@&jiNll9mJ!V#RPHtX&_z8=OOMR15P*PFTq(z$!U3&DHSyrhTff0z zqv0l_&Gj>HNkz@NEge1kj!ev)yRxx!@5#l@yDuL<|A9inf`^KU3m+*ZEqbh+y!eSq z%95w5sY{=!r7e4|p1%BrM#hSlnwcwKX=SZ?t)0F4jZV&*x4OA&-|6M8d#|6r{)0im zhL47Y8$THpZTf6ny!nes$(FCCrCYz5m2LZOUcUW@Ma7PvmX$kySyk=&ZC$Y$}lu^j8qGbeC;Fuoq;j3{y!mbfN6)~90m`T18xjJz-S=s1ELvN4lo%SfLRcN zqq9S&0TcYR#MtEQuy3QVvQPWgO@=VCeBR zNtnTuk-*>TufON7{eKn{8yBCDn3SB7nwI|g7d^dwUylD;Vq14o zQq$5iK4;6GBWJGMdGhA_uKN2Qeq0E+7U%mhOThrhFSoY`gm5{59;gQiXLVI*u zdnaUDdj~vXxgC9nOT5VFMaIML<<0h; zzd=%_6wLt50M7tM5|WQ}WFr~5NJSiAOYI5s5fNAqEkMKlEV_dALIz<`9QBv|$ZtI71o65QZ;wVGCKfLKUVE zg(oy&2}w9Y5rz?x9AXfE*ux#>@P;*48Bch=)1B>P=Q`DyPIR8roaH3vIK>%G zaDLO9-Q?yrwV6$9UelV@q~rZGl%*u)C`B1cP=3;ro#f;uHJM3FUec14 zq~s(e1^1fQyViBCaeZrB*P7O|mUXOQ{c2aYn$@dTb*fQ)YEze*)T0)4s6qW{Pj{Nr zo7QxuF@0%ESDMn3mUN^c{b)xwn$e3^bfOV`XhRp8(1R9qpaK17Klho>d)9NF@qA}H z*O|_9mUEop{AM?|nayifbDGh7W;2(W%wraFn8EyIFL#;CTh?-xv3zAKSDDIFmU5J# z{A4FLnaN95a*~mJWFr@u$U_!#kb(STANQEYJJxZIaeQMN*O{9+fkn8hnr zaf(p{epCFU_(k!9;`hYQiC+^xCVor&l=vm_L*jSD&xl_UKO%lZ{Dk-g@dM)b!_S9b z4?iA$JN$I`&~@B`rY=jZ3QpYL1W`NsFX?Oktr&s*N{hWESO z-EMZTTixkK_qokoZgP)X+~EfIx4qqMZf{%L*~a#@tzB(uPg~m2hW4|a-E3wrTiMA* T_OXp!Y+?^v*ue(&uYCmow^qet diff --git a/tests/v2/fixture/test_format_compatibility/array_1/compressor_3/1 b/tests/v2/fixture/test_format_compatibility/array_1/compressor_3/1 deleted file mode 100644 index ab304f14c3fa18fbcd8d7e80ff385a89a62f882e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 798 zcmV+(1L6Dv0gwW)1ONcA1ONaY0{{RN000060{{RhwJ-f(umj~60QSGU4iNBc8~^|S z0LpEd;{6tUeL{8j9P0U10yhKZ^ zL`s}QNsL5Dd_+fVL`GahMNC9QJVZk*L_!=yK@3Dd{6jzNLq6O?JDFH_r z%1?H3lbO6^B_|olM>cYii9BQ>2N}pe_HmDSyki~b7{@oZagAv_V;RR7#xHhpi&?y4 z6{i@*CpK}3Njzc^hZw{k_Hc(eykQMz7{eE~aD^#6VF^bV!Vh+EgBiSF1t%E62R3kl z2|Qo{2N=MA_ve21F(2{q9__In>2V(AF&^RZ9o?}V*>N4!F&)wI9L=#D$#ERTF&x41 z8@;g`xp5n{F&nY*8m+M!sc{;m1xFj&&vtgRnZ0afCmY$vHg>UzJ#1kI8`!`0b+387 zYhC9W*SEHHt!X`LS;rdIuXc5-S-omiryA9#Hg%~HglQDJZ3S68O&eya+kThWi4kJ%U8B?m8m>sDFJDRnZhh z(GxY%5+%_Q710m{(GT^|4&~4d)zA#Z&8qmCy)<&_$(7(2q^=sex#`nGLU2l5NTi)@8_q*NQZg#I*-RVa6xy@Z}a*tcw;Rg4& cz1?kYZ(G~h#`d+XU2SSlTUr4C|6LSF0|e7mtpET3 diff --git a/tests/v2/fixture/test_format_compatibility/array_1/compressor_4/.zarray b/tests/v2/fixture/test_format_compatibility/array_1/compressor_4/.zarray deleted file mode 100644 index 86c7d4c156..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_1/compressor_4/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 600 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "RaRG6Sz23MU0z>cVPa!sWoBn+X=-b1ZEkOH zadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^?m6n&7nVOrNot~edp`xRt zrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T$;!*j&Cbuz(bCh@)z;V8 z+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0({r>*|0RjX71Sa7F>i9Ut J0#psCYYMVGi*x`0 diff --git a/tests/v2/fixture/test_format_compatibility/array_1/compressor_4/1 b/tests/v2/fixture/test_format_compatibility/array_1/compressor_4/1 deleted file mode 100644 index 00dd93a5e2aa0538de48f2da6195292736571133..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 318 zcmV-E0m1$P0g(c*1ONcA1ONa&0RR9L0000c0RR9fwJ-f(umj}?08|iISz23MU0z>c zVPa!sWoBn+X=-b1ZEkOHadLBXb#`}nd3t+%eSUv{fr5jCg@%WSiHeJijgF6yk&=^? zm6n&7nVOrNot~edp`xRtrKYE-sj922t*)=Iv9hzYwYImoxw^Z&y}rM|!NSAD#m2|T z$;!*j&Cbuz(bCh@)z;V8+1lIO-QM5e;o{@u<>u$;>FVq3?e6dJ@$&QZ_4fDp`TG0( z{r>*|0RjUA1qKHQ2?`4g4Gs?w5fT#=6&4p585$cL9UdPbAtECrB_<~*DJm;0EiNxG zF)}kWH8wXmIXXK$Jw87`K|(`BMMg(RNlHshO-@fxQBqS>RaRF30s{m91^@*n8896m QP>cj9DSWyB_yg3M049#FnlOr`2U}Q;eS1l zLIprUki>s=g#u>H6@2*)t2@R`7R%I1M$=(v_ z!~F}+b;5*Lml*!H~-sYydx(vhAFWF!-r$wF4Lk)0gmBp12K zLtgTcp8_OMkU|uu2t_GIaY|5W(;F_il-UJcqZ@+&oYroOlAtt@jNf^B2$^h zbY}1pFEf)_yuz!@W)5?i$7{UKd={{fMZCeAEM^HyS;lf!@D^|L4)3y(Rjg(WYgxy7 z-s62fU;`WZkdOG7PxzG2*u-W&XA4`|#ut3aSA5NOcCeFO?B*N3=(^}THj&-ePeH+-&MmDyI zO>Jg#TiDW8wziFJZD)Hs*wIdQwu@c8&u(`2ejl)hJ$=xJeAr(0wvUhasE_%$eeGv| zpKyQ!9pqq#IMiVd_en=M(ov3fjAMPuryb{bC-{udI?+i^c8bsWyf65oQ=R5?XZVsY zJJVUd;;YVfj&q&oYrgJ$7r4+xzTulLc8N<}=5kl~mT&ux@4C`eu6B)UUFUk=^L;;X zgB$(OkNntA{M66fQUiOMtz2MyV(rWuk19i}Fz+Dn_NK995!fREz3SBWgyis2z2p tZq$qV(I6T|qi7sWqG>dX=FuWrMyqHYZK7?oi}ukWI!34H99^Po{0DyM5(fYP diff --git a/tests/v2/fixture/test_format_compatibility/array_1/compressor_6/1 b/tests/v2/fixture/test_format_compatibility/array_1/compressor_6/1 deleted file mode 100644 index 198d631f844ddcef1218f450c2aa08b36bb5282f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1071 zcmXZcgR&k507T*c?%9ToZQHhOr?Ju4wr$(CZ98e$25oHHO*-9~Z)Tt1C@8+xIuQ6N z-4Brifj~9=R9!VxQ?*oEbyQdNR9_9$P>s}BP1IDstC^asg<7hWTC0uPs-4=agF32{ zI;)Gis++p2hkB})daIB6s-OC6fCg%i25X3hYM6#=ghpzVMr(}5YMjPvf+lK`CTohO zYMQ2NhGuG(W^0b-YM$n6ffj0!7Hf%?YMGX6g;r{nR%?ycYMs_=gEnfDHfxKvYMZue zhjwb0c59FJYM=J&fDY=A4(o`H>X?q}gih*|PV0=$>YV=2d0o&&UD9P;(VzNDS9ML- zbwhvaAKla~-PRr5)ji$U13lCuJ=POF)iXWU3%%4Uz1ADO)jPe{2Yu8hebyI!)i-@t zfFJ~zUkFBULJ*QrgyvVm5SDO+Cjt?PL}a26m1smK1~G|6Y~m1?c*G|G2}wj^l8}^S zBqs$aNkwYXkd}0$Cj%MDL}s#(m26}u2RX?_Zt{?qeB`G91t~;ficpkd6sH6wDMe|@ zP?mC(rveqJL}jW_mEWjFb!t$PTGXZvb*V>v8qknNG^Pnn`JHAorv)u(MQhs7mUgtK z10Cr^XS&dpZgi&yJ?TYn`p}nt^k)D&f&zi>-{%cv5Q7=QP=+y_5sYLMqZz|k#xb4= zOk@(1nZi`2F`XIAWEQiT!(8Sup9L&r5sO*EQkJot6|7_xt69TZ*0G)qY-AIg*}_(~ zv7H_4WEZ>H!(R5Wp937^5QjO!QI2t(6P)A}r#Zt}&hZE5xxhs(ahWUp$zNRM8rQkO z-~7W(ZgHDC+~pqkdB8&+@t7w(PK{`~3qd&q<@r`98-zQ>N`1 za?3nE=REvwT>g2hx4+=?&d0uiO|O*foX_X}60a7#f8Y3)FZuinaIsup@pF8I$lUtB zeIvi-=lL4W4~l=XKe~{gYazDmJLlQ%-!J6nTZE6T(XL6e7V&c~!gE~;Zg0MHF+cBO z^nLuyZC6z<=I3654l|!u_Y!{oCHSo{XJ+EBC0vK4@cekjp<7lg<$5f|j-JnqT5#(! zuFEofHf&$xcm8EupXC@m@6>+%-&xLeT8@5I!*{JZcLmpL1-cyFrS276w-q>g)xWbJ ze|RO=Zza6X4>`8&o0VM0RoJs=`%C}LT*dWV1zC=stGTYL@$&H})!MA)`mO=`=oMMR zbzXz+ul}L#HC*qt_+!;$UsNq!%XMFipn7{-j#`W9(aN@4x))4SX*f@WAFnxzl%S;QQHth2t+%_eQ>_jnJR&!uK}veQm^>8z=u; zTfLF*Z6gw88w=k%e-q!|Cg|UG@e7;y9yg)rw`09Ldp7ZXZo=hf_e@UtdlTR5W-Pw^ zQFU+T``wIFF!53Uip_k_oALHD>rR>R-DbY;El5pt7GHbY7QXi_82FVwv0M25x1eNa z_bX=wws0S8!Ih2&(|s=8%KfkvOVjVDdn@yUeK-FA6xNm``Xj`p0|zrWE(Qq z|61De+&1o)ZP2T+aQ8Oun{6nQg0hYKXFIN*_Gb2nhqrSdZO3xCwsSvi#~Hb{C+`1t zJNMNNOppz_gZpa-jBh(uto?Kc_t_3SDGRcL`)voL{Pf((eYX?WN`ctP{kIb% zPVU2188F!`Q4&U$I)9`5Ho_~z-C zZ0Gy-a9{6%l)H=fa)0lI?aQJY$G)|f`+P5MRMoU&#a{0By;y5Md!7C4ecbo^aCYj} zVQ)XPkNbZgbhYljb05#aK78MHUXAmA`* zT&=l>O&|5pL)NFmN$^;IA1l0w## zLfpT-mnzAHtSg21D8JbdCKLe`x^j5v6% zx{FwUim>4KO}Y2pTEse31YOORJy*nfRD|OfG>-l0(<0WTBHY|#OznNUi&&qEAlt)V z#5z@kjeX}&{q@fx)~jMfAGk!_#jIP!(5HRP!^Ny$#dvVEuE@o#W5tkixV)J4tQh+A zt*2FF|M@#XuH1smo(=zmWqVC_* zZz*G4EyJ^Nm9f5-VbSYa!IZJimO+UjRj|ugZ_6O%!&k<-TZY?ZLCRQv%OUN(XF2O| zIkt)bEoVJ0N3!TGb(ga)mqXg!vU1kva;WCJ_~nXn*6DJ5A`P&d^|~Bdy|q23f_1wB zzy7HYOa<$A1!Os%sbC$ifL3Q6W>&DCS76B{olD->S;4wq0VUQ|0jXepuYguxo&Ts{ zov(z*my0V|?*rP#~N;`VxOpjR!2RbsbasV!d2DnPoMlr75hdNMm6qG zcNP0b6_&;CF1yTI#XeF6B`%eNsA4~_-!_IzZ$JHW8?d*GY zIA#s{=*V?;_CGsDtM0!JcJ@I#qNu=8a!ZXhfu?QTLWpALu%M}YoLm=I)F9ozctWmV$er5?87z4 zJH9~OHSEVVc=FSc#`aA$?8`OK-F--54f}HqW`F%lzkggc?9(-n~rzpjC7k3%)= z+ckJv3Yvrc+X1mhT^#J=4(RST{5l8wxdYc0YlY%qUw1&{!4L=gy92r#jd;z$KJS2P z2C-8=aU*Y&5R#&dZB|lz=3hE>I&(g9&kWc>t|99 z>H-JkxEyj&A2^`kfZ8Fo)Csi^``)FNdZ8ART3@H`TIz;c=-rWfZ!PshEkq6tsilsn z#Y3yV_@~WlwbT=}(5sREVJ&q)MD-<9q%prt(H2Z7MIC_)KQPrLF{aoI_i=-JR^pxj{2kyzrS>^y6dP@ z>LBuONFDV`9h4YV7NL&1r4CLJgmu&}b$EHYJ~4IFF?CSQP92y!>X|yoaun22*VIAN zmv*i?>YF-9`$^VO=hQ)&NmW4VsCVk1-GBSj>#2L{AvmLRJ@rpLP8Py?>Y#dP8q)FJ zdg`Hih}<1qPhC`xb`M>&b=<4<)JOHuG@#Rm_0&oAcw}hqf3`2{sh8^UO?>JNZ*Huo zZmNf-J6#Iush{f6T?|J(byPj1-6rd)r|O~Uz=glnQ&-hP!6Vgzozz!O2!817q|S0e z)0b}7I;ppuQ07TVYA1D<6C&>iJE^~%(6pfYt4``LCl+t+eTDM_C-s;UN^C33<)kii zLesGx8=cf=PRMb|cT%T0p-;<1r;~ck2~7`rCY{u6PV{yEs%|Isn-h9>ynmXDI?e^b zX`Nlvb1s}c>Pfrj8W(k)3wkvs-{Ydbb3yFPU>9|s3#r4JIJv0zT=+mP7j>Tt9sV1j z6t#=`&jqn#8(h?ZEcs{KF6z`k-PiyHmz2b4pnhz?d;tOt)R7Gk`!=Y7da?n^j4B7!Kwa4YS&k1Hs4p9! z-9u=819fHt#BOeApx$hNl+*kM>dppeH6N*Op#E%t$j3wjb!Y>WSWybDfqJw7T3yFZ zbyJtRA$YKpoBGs^uoyBob*dW*b|^{Xre1YJ?CKylb*mey*&gowikte?4cQ)3+|;pd zWOs)ipEKW0J?nvCmlJO4T{pBk%lzV|?sa2U zV2MI@9_n8Y1n+h7PzQUU)l&9q5B0DIBVIdt@Qb@W)Wsf%Js;$uKK5Y2#LrdB^-w2! zAjf5jhkDrqeOmg?_fR)`Ah=+?hx*wAX}5VE>Sz!2?zpqwLp|+**n_x-y4r*0OWUiK z>!H5(K(EGKr#4b&H$rf7$42VyMrhi$=;}u5?nc}#S0nXzBce5$_%u?7HwsN#{Bk4p zcq62IOlhPpZ-i>LYC#&Q&l{m>+0ylm)ai{7`H|O1z21mvKh~+ck-EJRnuaZlH&VYh zLYcEz|6dxZ;~Vjm7^EiZ`6dXC?bt+J--J1GV4A4!n;^^4uZcRp37TfDe5#3hzX>CI zcKhl6mz${jn;_+MaufA`6Ev+_{Y4XfKobP_tZ$+pXo6O!hjN?f3!0#5)S9{``hzBj z{fjrzCp1B;u_He>(JwSX)24N&G}AXULvVY?X8MO_X!UXQs%H9#W@wtUzF#x_L^Bl3 zZ#(d*X8MX|=x+P_OU?8b&5-soxtTtr84AWJK-o;c(G0;^>ze61nj!Kux0(K<84oYl zimaJFq#0s&W6ks<&6u&}XLUEzmo%fB2r@7Ii5G&;J9z0+ycjIN-AljXg{E19`g!SF zywIoR_>*4x7cUg-Rm9s%ALE6lOM@qS>1VvqyW_+>FMW*{g6r0K>2JKyv}b6pmp;b} z)oc}T_R{ZoA@)AzrSI`V)0bgCd+C3?_)jh`eUJ|dwk{8!<)a_+LE@nfKKddbBp&MF zqd)RN(~=Qa`RJ2;5PNZxkABGqDL4Im^i4i!8WMfLNB`u5YSv2O`sky45Iix`M?d9* zrX5eeRKKd&kMBYsH(P#OfX-4d0AN`gO{rAsPw~xNd2f^PWi-{^;I=a2pLkAA4|K{X^leWV|gbX>8-Pe190 z;OljM`bs|(yi$hDPk-r$*yS8QeWo9Kbo(sx({K8r!X8EJ{PdlEX!Ut}v!DLc55Z?K zKYgelmZ>j4{iq*W4Hp08r!VzGP$64q(2Wrr^TJ_3eu+s;g^ORq+bt0!5D?8gY@k|=(KD3(?R<8Amq4= z3DU<0ai%&g_l*zI&j+E?)GqG_>Fa}#b~GbMe;?FhQUUiMeSQ!p1x%2BKM0A3)&}YO zgV3unV0)1MKM0-fUzigl2M9vsZ)uP`AP9$CL2`j0bQ-p*DM&sLgy8FFkenb0DIaE# zydVe_W-A2}BsT~`?DnxB`9ToUe*X=UBZQ#SvDIgW$P+@4ae?+Ba)l5i{=X= z*SlXCB4-Fe)2ze2L*xx1h}^$3MD7rRPRrKbA0mGULDQuPPlU)JLJ&Ovbcj461f3T5 z7!x9w2tm`HBjZBk6Cns*m=q$X2tlW3>t=+=D?-rpW#Z={a*GfozFiz5zX(B3FI>Df zM2-=H3Wt>C4UuPraGo@n5V=MOI!)V98Y15aLAHk@M9vYyA7TJQv6eOh`w5hhm&L*kVYVe*wQ zq}{$4CT9sl4TmT|5hia5L-5h0Fu6+@A}42r$zQ@y%~nlpgvnvTkhpzOm^>y7iC@=* z$z{UWCRdnzCJgP4rXC2B(}bbZ$3dlG@|rNDd^p17HeqPDa-uOzeiMdH7yXeiIZhZN zKa4PWP8ixfO#3lRt`mk%4~HBJlkbEf^7-E|IZp&y&HvLTLf#XBjK8#tko!a+<0#!D zoBr+cBtBjiL8NIW(oLS7WX zcnMS@DS2f(Iu>$dMutyD%d{o)p1mF}M+Or3hp^Vo`*ADFTVh*F?yf zBGBpEh;0$_rU>E^$VbSXA`m=L8X6+v481`%?p2y}Wj z(uj~xMWBX%lp>3eQ$-*+^jL(vDuRb4Ad8S&MWMnJCCH-WS5e6LPP-^MR`ldJOZO;w zRupnvu85LrMWIhitKL!attbS?+z}<`ibAIkIRm2PT~R1=r4oQBxmOetZ;ptPe?_6w zfSX>7l7mH|SEKc~D0x^Ef`2DQ$;F}&J2*W`J{HAXxuWD`QOG#Uq9}P;6f)khCQ5D= zg_MtNQS!4Wbh?z@A06_#J7I4AmeS_W8`x&Xti!! z9wVoVLDSEo8)M{kF$iwEBSvl)gI1?w2E@qkVo<{ds-VWm@nVp8dPIyoF9wOLM#sqY zVo<|&3IWB)_hJydKQTtm7lYXS=`r%Y7&NWBV{VMxF9sPuS`;Jyi$TUwR>#NzW03Z; zHAWs7gMz&(5Qvcr#vpjHBt||M!$|=WBPWbu!L3@6#mEa|koY|uBR7me;<#*#{4fSR z9kuX>7&&4L5@#HZktfEWX-~g@V&sZ(=xMihZQ|sMamYAa+c-I695No(El%DThn|*N ze0iMQF%D&JRU{NAe~d$L>K$=%$T;+LV21&5^2j(;vsDRaoLn*v87~+fC!dT%Pfslw z9Ve%Z;|;muq$H5PWqaPEMMDl+)G;^3nuk9J6hL+%y3hkL;EpKTSa7 z*yRaw)CBZ2W!D=M8g^D$G|Cq)BqLg^GA(JGp zPC~2otp}6j)=5a*d?ZPJorK`$6G?LH6hw}-PLXG)poVW%K~0fsry%3dT~p-SDTtlx zl_KX(K@HO=B$^`cPC-wz4epyF_fA1@(0wWL?-Vq>J~1#w4xWOHXAMh{ho>Op8lzI= z;wfmFJbi46d^`m`jW%RLikv(J!R^yhCH&$%gGqk*lX5@m_I?d_4tCKmW6($k|iS(_&$Fio87q89xi9$lX(r zagG6{%P|3G^BhCOq1)UA>(z!(&YPT$hgg@ zG&z47B0t8a$@|lg@rem(a{n~+v{ig+n*2Wvu`jdJ%mLC+!{e&urkMw%A>*&B(#!?Y z(9_hGRzw?(9=vWT##Yzkb#Wr zcg-+=$UtyVuMBgD4D_^8=K2gy&NXEqarNyP<`NkQPVS#!K9Rw3HH|cSV1_wG1~MKw zEW^Aa0~wbZm0@m?fu`%m>lx-38Qd%b;Th%_8Ax0O zd?SN5WS}U+oFfBGPtDC4<{cU6G`x6khPg)u63-W9n15ujQ3h}`%t12H>2zOrhIvQ^ zGX5FLFc--{#=+7V<|7$IRa#u~eTF$n1~LwGIK#Xo1BnNZXPBF0A$Gr2mib8*=UvXJs|eU|x377xmTWSO&Mq0`NN{j z+$9ScZylOt{*uKs*&tcwFj?qn{6(*4na5-y<4^BpnagA$ap=@6^O-F2<-lZ_(_|sb z@l}?2O%^g&?$v&?_8(BJOpbUe!(C`%t?Fb^{5 zhYaRI27Qsie8`|bGMEz?ct-@N!Mw;oXBjXzm>U`NO$PHLgZ{~2j%3hB8O)Om`YD6C zl0jc(Fkdq0uMFl)2DXc!H<&l6=N7+ZFn2QOyA0+}2K|@89Lk^%Gnhvi^kW8dDTBVu zU_NEgpBc=l45Y;%8O*EHbBkXym|Ge2Z3gozgZ|B6j%Cot8O*Z``Z03?acP9O-$sEt5k2RU+ne?+Jb3K#3)?~hC(%+iQ`Aqs;lX;(s725on3geHBm$-GcKw|rs7WNv8EH=E24 zP5Nh(Iig7)Z8A?Z>8I6mOHg9cSDVZiP5Nt-IipFRZ8C2((MtTb$=uPT?>3n~n)Kf$ zb4Zgu++-eU(vPd>J{dgGmz&HdP5N__Ii*RTZZfYlakmWEo6Idu`gW7~rAhy8GRHLO z<4xw7CjGp6ZVB>C`g)W3rb&NqGUqhu^G)WRCMJsCH<^2y^!+CDPm})NWDaVQ1DMQ1 zP4WQs+>(o$AJ7V}t(Ji=lwYmrM> z%x5j~35z+cMNVNcueETYd|||5ZflWSSj=xN@(YVOu0@VvG0(NgGt_fSu4|ENSj=}V z@(qhQuSL#bG4HkTlyrKFxvxd;VKM);$UiLRz!o`(#XQ&|4^hv3a&Vqp#9}^dk&jr+ zi7j#xi+QnyIr0TSi@C9SZUI^r^J9zr#A1$Yk)v45lP&TT_1uyxTjVMh^JR;C#bVBE zk+WFLn=QyVX0gTGSv|J^LW}vcMgC$jhqlOJEauS`d5pze+9H>+m`_{eGZu4di=4({ zUTr~tyR=8vVs5RTTY#&@{MsVFv6y39Pa{VysZXUE%F_UIk!d5)55&F z1$z2=QtKAx-Yp0Sz;0pw-9rA;!W_JX9H@nPcnf(@3v=-na-kOH<1OSvEzHSV$cb8* Sm$#4?wJ)o1Bq>oDk}@Pi977XPA<9@ZAt?># zP^l<|RPT8%6oq{EzMlP0eAjy3>t3Gax4rIr57)k4pB;lfd*|$y4wh|ryw2QK4xX%W zu^b28`t1pyzQn=V?=0?g1RNYH*Kg(E%N=}uQ{9u3whrbk+BWjG4h|mvd#bnfDhJo_ zb8zacvMWBk&cW_p}(zYXqI@tW8M&jVr z^f5UScbf|<^d9A)*dH#!69sKxLwZm5|khnX3(dXTT4o1B=_J8NR;h;+jxe*6vw)tyDt>q35 zOnOs}1HFwhs~o)2tl?*myyIZV<3BaY-*cc*+_6UDKKSW;oyf$^kp0@;lns62}9y;=67!9J81OyocHF14*uS(3F6?( zdMAu|CUfxegB#X0QG1Rd=CT+P$WW#hXu(<6>mJZ;$VAii^&Y9ji}uamL5(cb;3% z1)rt93w@Q^XSkTV>Ve8r&vfz7F^%Q8(AyYxj*Hq0XG@N_$oF@f-|u`EA4wEDTDX|; z!hZ+;yOoQ(k|)0V^dc9R@N-dR(jPCcxYUKSQ!~{C)1$46rx}Qg9-Du>aNJcc&aF57 zrGp(^lzs3zIWE5Eg}8X5+J|4=dV`C{`aIh5n=USHSjCq?Lq3eetzq2iV$*9q<+zx$ zzvlUMZg)%4^bQvtUU*tc*M$`A9V6~?k+nYb`R;pMtY@OSn7;FVIWF#Osukm+^~hg) zeg2S($}AWcQnZcVdejB4W4Md)AAR=c?8jVa)m9rNacj9e>7vZ4J&pbu<3g)t&{Hm? zXr+O;81Y)&n|n@jp;`LvGcFoj{U13l_CJ5k(;rNCq1n;=Sr;$0{y<9C1)pV>3w@R8 zFS$6WX*{;V%Pv^9ue#9N*gN0Fw9`&}_wI!f_XA%^>AKJ;x-4;VT(v8r|1NX!%N++g z;Y}AS`MG%N=m966{f>+7-855OFg@0|kfN19Tx`9n^NPbCx?ubG*v0Vp6FDxf=Y_ba zKk>q=w*Jq>z8$|W>G*{UDO!0MiJK4Of?;fN(eAYG<+!Nzz?9c#{NRFVy2Hh~6Us>G zx{#vX{>RTQO4^uBE_n@y*siudi3wj?Ob&>~`ZEX*F8~so9P>GG*1KWCi4^p%T zH#PJycEbC&UvZ{~n|J7d@o)}54@W%a|L2uv9=3d`nd*V*(Za(p2I8U9`i*Bja*+qN zkJcXk=zXai51;cwJaD|U^>BaJd!%$dv|XStE$UJCvG3ZO^O!An}g1SPW@raJbZdl zyH|c&;eqXAr3an1+u!ye!&WZD1INp14?5*qf8b#?2fv5Od>9WpU0(deL(Ahom*YW( ztxP%(+SLzk^02(j-hK~!HGx$Pe2h1dM|yq`Qg zQc^I`?s1Caq2xL|>~4IXj9d>|wZmf%ESJgB92N|}~$UJCv%s#?L`T32cbbatyj`pFia;Th-C!XyrBi9GZwvrDSwyS$o z@o{jxHg+Fu>(zZo(TWoCp(?2PNj^CCYx_8rpAS_Z(@yj8$vVwcA54#iK6L8uY2*WM zt~vScvwg6AH1VO+Rt{v?%7ysgcxmB7r`*a5e6--;_d)s5+J{b;>X-UhI{q>_K4jR6 z0P&$+{h{_g&K$0aOyb_uuzjzNKD3)|xz@)kOjI93*8N+K501GmKI%}A`S@*x7K{(d zpzc0+9X%y(3PT@SwSV;X!E)*A!<)Ia!q0d5&}!*=j}Nxf{yut**G}d`vvk!1J~(b4 z@*%@k8krBxjv9~pP<1+Cm=8Y7V?Oj%hCJ>=Rbldk50>p1iF?JRExsG)L)F+dPy1k7 zpXfu1R+NwrRY7k|^}(?}-N#IRK2&{FeBK9T;fp?)9QYBN!N$b8(*g7HBav_<09>-f%xbBYQ> zAC$@4eXv}1`sh)mlSpMBS}kAv>O<9fThE7xWa(r+G)tStK2#;Yn)=XTTdVNVP_kq4 z?>emWXBv-tpb^x}IIstUrwmU6=3|mpC0XSY7 z2GA)tzfph&9Q*+&Kbi#4>2h?l0CR?(Cno^qY4ZTu)%#u$;H3LhkpgFhxZS_XBiMcU!~sz z61U3rK>=8{LjuUKmBk?d*STQ<*w#k`&?p)`9zaC0EHD8$_QwPm$4}xG^>J`~0LsFN z0hk_>1L)N6F*N|!hG_xVK4u2cX}k8h05WVvp$6c1nH@lp`@KpC_l0I%b-09z;w15hS!3ZPrVmM;Uyaw{8M0IpHr22i!$;ya0Z_dU|d z0%(@j-5x+y^7NenWY|h03n0s_D8m4%PWQS2N}eTvzRIpB0OfibfMuHpkYOu}LjbOG zdxDa-{$~JKJf{?-a1xHeP@!S+!( zgihO*$A{orSuF&|%ZVX$%FU=1LbvRSwL?&T)Cr-}<uV za^qPcxV|(A!8X=3gl=<<&kMozws{DSxeG$baw~!?#5XLM5R^e}Lg@CABE+>6h9M}E z+l64cTp2=^Tan5kxJF$oakGJhkX>IoSqQG#okOTfmIE2K(#S$+cI>+)gsRi~dW7J! z^a`P`a%1lhlLVK( zLfwx$MucE`j0~Yuzta;T)OGlHbO^SOaUpctE*&32xBdDPLvXxI4xv-7^3)Kt9j1k# z{Fo6!r^|rnLU6l%LE@$W3Bfd-6GFGjcCUos`Z6yB+t_O%beo&ECdZZCb_4e<$uVF=3PwIOtC*sv}{i2)*UQ=o=WwchZv5brtK$wF|= z-V{Pr@|Z6}$gq`07J@d+w;`xbw}#-eY?HXzP(x6z?+n4R-4#NHtvEsEr(+Ch6lsQa<~j}T0c{ULPffBJU_bsbtC3c>bqSmNfujiB59 ztg;a}Udlz#DL1x41lkT2BT#-+j=c?@IRe+0IuY2$ z>PApg>Xg$XaJ_91fn%;w1X*rHDo3FG(KrHSP}2yyz3e%++J%E6sOzwDu*A)UE`mEO9unFY&LEXX$??>R8y*7fX( zG={46$>+vUx3Em}7+kYk#!!` zsRmcappAP?46bvXVz8}WFLBc#i@|ljYYdM48)MwUF9z+PTVtsE(Y{vBgY#FgOO~$HNjg4b&L4rH02)cfIw< z7);YqF=$(kj-jshi(_M`NUl0QhHi6XC&b`-J1GXo+>{uy+=^6=p=NUT88Ik>o{gc~ zONSR?sC&9*Rt(DIITAMw_87EJ=f&U}wIGJ7^>T}1&|ZHd2G{H*F;perxjcrtbUWUR zK^taOj5RcbVyHWI$@?*A)2)d?xxOxjx>B>($Dob7AqLmEPbF?`>z~J<&A%}Q*ZnVI zaO`i6@eRKiw1c+BQ1@f)_c53rKgOV4xFg0MaUJUHilNkF+AlFm?Bp1_?LX$nP};C3 zj6vLx#GvhvN!)yx7z|@~4BAqE#GpLg7lUc~cMRH=2Vm{JQUOxfX>@yOmO5Sy50^*m(324KdlYnyU+yv^5tvWveZMv3;aiGLFP+}ZN;$~w{ zi~}WL6m1jZK#6gn#5ho594Ijklo$t+xEV-d94JA_LYEi^N{j;~#(@&!K#6gn#5hm_ z;;4HQ<3NdVpu{*(VjL(j4wM)NlDL_uiE*F=9CM=*<3NdVpu{*(VjL(j4wM)NN^mJT zP+}Y?F%Fa%2TF_sCB}gg<3JKO8(Cr;C;{cz!o)aGVjL(j4wM)NN{j;~#(@$HCV)wd z10~?P|3P9LC@~I{7zav>14-N*bcu1G1nN3e-kcZ*N{j;~#(@&!K#6gn#5hob#pFPV zai9bxwp3yqC@~I{7zav>14-NzLWyyp1YB?TCB}gg<3NdVpu{*(Y8)sv4wS-S;ikrc zQqVp^1Et1+QsY3WaiA1R30I$*8V5>^1Et1+QsY3WaiG*VP>Pe;*i+*` zDU?R-XptHRN{s`h#(`4fKoU0>snj@73fhGoQsY1}O}Wse#(`4fK&f${)HsmD&B31< z2TDO4b!%!IC^Zh08V5>^14-Omv{U0iDQIioml_93jRU2|fl}i@sd1pxIFQ6mVVD{R zNqCSf<8-QmD({ z@|)B+P-+|~H4c;-2TF|trN)6$^ra9=jRU19S#MM0K&f${)HqOT97y7(@sJt^N}=?2 z!e6Ozpwu`}Y8)st4wM-O%8UbLm_?zS83)RslyGsC%s5bH94IpmloD-*S=4P{M&S&@OD183)RY17*g6GUGrJH;v28I8X-K z4p(KyfimMjnQ@@ZI8bIBC^HU};aD!(nQ@>Dw6%L?#(^^9K$&r%%s7z5O=CMV4wT_h z3d77eP-Yw`GY*s)2g-~CWyXOdZW=L}ai9#uFJm&}K$&r%%s5bH97y6O_{fX{WmwL| zAu|q?83)RY17*g6GUGs*aUhACMqOqcC<8I!lFT?zW*jIp4wM-O%8UbL#(^>fTr4x= zK$&r%%s5bH94IpmlorXK%@j17*g6GUGs*aiGjNP-Yw`!>KeLGUGs* zaiGjNP-Yw`GY*s)2a>pH^k>F_GVo^?2QuS8nQ@@pI8bgJC^rt28wbkKmBv(V94I#q zlp6=ijRWPzfpX(OxpAP}I8Y9CKg!q7jRWPzfpX(OxpAP}I8bgJD91z^m$`AE9JC!S z%8di%#({F<3PD_pxii6ZX8JBCI`xm1Lfd# ze3u&s%8di%#({F|?bK^j{aiH8dP;MM3Hx85=2a>qSfpX(OIf!o$ z<;H;u<3NRRpu#v%VH~J14pg8rIZ$C7s4xyx7zZkh0~N-B3gbY9aiGFDP=OINE(_y8 zg>j(5I8b36s4xyx7zZlQo*bw!4pbNiDvSda#(@gsK!tH2iJO9~Fb-5;C5`REI8b36 zs4xyx7zZkh0~N-B3JfO)DvSdaP+Ga5cVQf;Fb-502P%vMN!wt!0Eh@ z!Z=W29H=l3R2T;;i~|+MfeOSFAcb+D0{ZL1&MOMzK!tIj!Z=W297y8k!e1B%DsVG7 zP+=UXFb-502P%vM6~=)I<3I)K@rO!j(5IFQ6mL#QwgRA34@ zP+=UXFb-502P%vM6~=)I<3JKOIZ$C7NaE)AoBb{MU+KT4H&A*5r8iJ|1En`mdIP05 nPg;$WU diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_1/.zarray b/tests/v2/fixture/test_format_compatibility/array_10/compressor_1/.zarray deleted file mode 100644 index 6c4c6a48e5..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_10/compressor_1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 2000 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "MnFWxE-+D1U&O|u#R4%AOh8ZxMF|m6y1Se0j+vd^ zo!Ob)o$k8#u7OSYH$eTL^YPy2K97$IyLZk#=lkCG_kMHADJQR)Yv#Cb*mDa`{_}ne z_@UuI?I%z0--lw%fUEEMYuQNt_ZQI4b(^}!@c+Mz!*54FTXM;3{BvXB@ATpSMx8UB zfBr4xRdjpf+W)=7zxN&*Zp!@f(Yq7)_b1}cR=sx^-%jGsnT%brk6(P~xhed4AE4`A zpSnNf&z*{SA72`5_s}%{{OS06$b>Jal+WPr`53n?F*m=x@)Q2PnfT>|{u?v-dq2gt z*KU43blzwD{h#3lU*e9rkIdq8%tEKTK2!H>KF=I9ZMp8q&y{odTyrt|yioIoKj-rK zKF2*%rtKMW%RD~kJp684{&}mnzu@!E$G(A0uaxYZ&*%OUuNJ(2-}sg<`TPrTv0PvA zb9{x!-1@(LBfsY7`5Mj-ihr^{x{#l1A-3#0=h^PxFXZQ2gpaM!u1T{N@pCT1b6pB< zZ@zRfKks7nef-RAS5+_O=U##iGoM%Y5`O+A_^mK!X5z0UT!*Fb{CLKpTUIRPdMw3` zp3jV0aO*Oz%QAd6Y+vJd{$*UB|1N^zx?bCd@mdDz~)1_(|2s(``Lhn<1bV9M!u(w(4X$Y_cro< zZN!@!C;wYpy^-&2BNAmB3*S3`6W`w^=-+nn3!C^JH=*daW4%0kHt~IK!sTc8OiuZG z6W{A*EWZ3vb#LbT-HcN(@lpSZ&3w7bTk&rD+SB@; zw~hN`8#34bTH5m5Htv^g(5ta<_crdEZ77q1vW@#^JFcGgX7-1Nw{stD$8x#0b3bjz z8M(G6?*Dc>_tg$ekPW(n`)db`Z#!13{d5QS*$zA@3$lazZ3m?M^xVmPw-eV&f!N9Y zw-YO$s#Nz*?!%pEJ?{IbPMoug`*9a0&1%M;Gx~zue%}3v1~W@?QX0-^BZ;V=KkG-Gkcvsr10iF+{b$``JOw@dTHh! z?&m%D=INJg=lk|>U+;mGyNma7fA59u%c2{{zO|S8d@pWP)wE;9Uhen3SZhCfo&D^6 z-1qx%cIwt)Z$Gn-`+pyFweG%iAJ4%)eBX9njq`u|cpmm+_;vXoPI_=Z&&7Vo_E@@~ z=VL!^da=#HCTl;>$pLKm=q7a^;CVTK^Oij~;;K&$@Z220)C0PL9^m;ofMfCjb9jz& zpsQ`?+jDrHa-fPq*SB8C;knAes&l)%@0gXt^OXbL9lGw#;W^7eo2%}-{@q{>&sz@e zy;WD#9G<&e=<0jXWw|_mx!5ZkCYR?h7cI{%Tix((F3)2wboc7HGMDEv7lS8Pt2>wH zGZ&vrLCobj%|-RXAJi+&<9W@)|JKcT<-^!=4T{@AG*)zj;v2K^?$6p5r{M zsnZ=LkLNiL0lD&cuJdq~@%oL|{GG@1osa&~;PQFS^D*V`Mc;-W$>({`$9`Fme4hJ! z$Z=Vb&-0&;9+wUNGgg()I*^ZHy}ujU`@4MBgM7@tTfeaa)`bGpJlgi0)U5@q4+Z#Z zgzl&XtP=(3^U7T{pMO@sdQpJa-}Cm!?k-^6D8PEX9DxGXj{=0}{}r15R{`rtAg;1MOozOzon?mU7x#-(M z)}2C(IC!qQi&%e(u;BMix%b{$#5z<2UCov~SHyZ$gyR=9j{WJ=BG#oM+}vYK?R~q8 zSf7d@+rwYPI#q;?edka8_0J;Kt71eSxJ2E>tXsv|qIqPsawu%5PXFV=Qvgj>!m$NRHL)zW4a@OZ^sOG!)<%)9F>2iD`4X~W`x*S@) zwLPbTb-Mz;{;3a41?zVOWI3LxU>&c3R%ab%R<5+5>gl4EO7?|He7a)jp$TVKu|HIybnCg8m$pQwUXM?IgZV!x=uRn_fJpZrM` z`$iQ;HSSP%75hgOmc{QbyUbg~K2ilGE|r6*Vn3;d$g>{R>?_r{>(mpCS3FqF{!)$i z&R?V?L^b*8C4V{$wSTWSbK4nABfY;S+W52TDaWOnL_AMK)sC47GXM-|B7q30T&VFad z)gnmk?0a@NW)1r2$aQx1KRZUN?!OLp_CY(OeGIj;AKJ0}LamVP?2C5j?!Mt8JNu&@ zO;_j&W@n$YW9;4wR7th7U)piTEn4x~**ERjARETc{%MD7-;|ww)DBsWLw5F4J0?7F zhPrFmS8E{lt4j_0YYlciyGmKK8ur;5JYZ^vP{V#(18J8-YS?#cpo+6PfHmyDHPC8e z&_^}w!!^h|zChhI?8h~D^3##V_DwbH%Qev5eMn&q`*RIufBj3pe_S=}(>0LenyO*H zu7PZiLpAK%HF#PInuGn@0kKD29PHx`=Cu-`kd(jRHHZ-Ik--vQknMs9Mj|2yE#j30A)p@TZWfpM?u3hAI8 za6nhn3AGUW-ldj$p%#-`U#IR`>V{hA-I055E%ie! zL=Fw9rH-h@L#w~|r_F1%)DyMPtC9a};1h>XJG!?%eAo6cW9ra2blo(YOp^mzx4o(q-b<{6)czL=$F?G~2bx_St9hf@mnL5aF z6x31I)IrmicCI?=n>tAQN!C&4)Ipg^RY2;fcj}Y;jw+#Os`T~v>D4_&l%+^hA}NA=J&pwox-)JgStWN7YxwlC|cm+J9N zeCiEvZmg$ns)wdKT?*={pX$+F3`ae6R6V5KChMuE>Y?etg}>EPSJgwoBh`YP)K^Xj ze(3C^&T>N2mu}ZOskfX^=1ECvCv}$-BJT$~slS}iw4nQ|PUE7lX}ewO%Hk|oz!hk^mYHLZYTAd6MA>N zf0~Or&IQ3~on6#(E}T8;NxSD77j>NrdNn5BQssl%E$xv2MC_&_cf zb)O3z{u`hawTt@C1+ilrT-1RsJf_P>zKeR$h3{lRT-1dwe5%}-FX5s-bfJd`UKe$u z3#C_{rtSvn#Rdp2>eN8p*Z>8Wl*DMDer&*e0Rj!wkqr?0HmHGmvH{ABDhJg-UD*Ix zjt?5BFB_oULuh^jb!G#^ZfL53Ds~f7>9`5~$oBGua z*&b8e)Uj@4cZVLIGv7@;>xS6r4Q}dMH?$f|<+-VE-B9L58C*AYt{Wnk6K?8VH?%s- z{NkqWbz@dwi9&WB>R%58?{)G}2YaB^Qub;O^{@vcUORg5i@QD4#U6+~ALOAv_F%!p z&sEFyP$zpJ$7PC#df5YgTKdlSP&a!ZxM019`q=|%w|O4wXb<%6xU=3vJ?(+mgSdyf z+Jojx+pCuAp}zJ&uf|=cHd1FdLU3}&M(XWGXxg^u>PG7BM%*n|BlUM9qBWZMG*X8* z3Qb%5awGM4BcyyxX{0W1gle{GK^m#g8=-00()Eqh>5UNik=IDQ-iT>G)~UOZy1fya zhAoRXQolDsnX_2`UmB_78}XDFq$cY5CJ2u0*hF35ggJ6xnyBxaAj{FOi8{Xtnr5wh zs)>5P2_t)U`|19do2dJnAmwy&6ZL-+G_6|wMH78M69o6HZ=xS)f>x)8a+~N2nxJXa znz|ami#O3HG(oGeBR@CMFEl~Zrgf(@(>F9jaC^sQ`iEv{^>Os7X8MR`XqvRX zUo-tgGZf5kJMgJy`if@gZu|R7&GZ+|koGdUnLeW#3dSiw*-XFD48d9Jn&~^5A@Vf0 znf{|04=>k>teHNf8De*1&GaM9n6c$&bvM(OG^3jcGB5p!7lO|_c(+VcZ@kd7 zXK1dMKF15yY!z_!((ia7_CDsN@9{#@mtjA9>3_WVPcAQgkPiyBE)SpOqaX4?;-L;c z`XV1B9_rzvKk`A-k`Y(==#zX9dvTMGe#r+ZH~oC{O+IKE5`Dl&|Kx*e)=J^}=%aiP zJTcNoKjnj_9Z$dHqp$MeX%Q4Y`YRtq-c0t(ekG{(X!QV@K z^j|&*4qfM?5A)%g4|F2sqaX7@>{PCgzRU+rBNAmk`ZFIoNMPopPxIl$+0E+q(XaU+ z_$B6}Z}UOBzdn|a{>_Jha^QXRaXx68ko?_8Kj(wI-v4~`b$;9;g4a)f=ZC~O9sKlp zen`C0!%x5G$3-n^KtEdV(-qxMKj?>c`=g%p(--<7 zcxj}c{?HF4Mij92(vewmNkofPS0R3wKS}m1c8K931V32H(0R3zLA{Xuo(ANf_)ytg^1n6%AP_RY;y#Re~ z0CVNQ1n74IkmVQ?pzjSpr&SBb2k3tT5PR}|fIc{YDgh(``r!a{+SqnpfW9~YiI)}! z=#K-CxM6L8J~@EjrJx1qmje(Sk`thB4nX8lS%CgI0G$T6uMN;g2XMUv3IY1*00e(V z1N7Aa$o4P;^w$A&5aX%GSW?ErMT+2OwceRmKlj8hgZNdFy##G~zl^x;9Q z7T^`69}hyoW`$gW^yNVa{^=8>KMz8u#hva7(x(UEmxddpUk^gT7=@^V^zA|Dv}^g( zLHhR~w}PXG$TlVAJk$}0rwz%eh?=GOpty* z2#JT*2I>2Q(5o?EdyxJ=2%YXB7X=$)1?VdgvcR65Ip~M zh&&<$ofh{P6C#%gLDQZi<3i*UAqZZW6e6bxL8oWyW`xKqLeTVO;^!f9ix4EfT^u66 z2tiLTT)Z|!juCwud7`&Jn^NVgN(r9U(~k z5Dk%ggdpw32$6q;px~AQpdoUQ5Co?j3z3I}pxyiAe?#OVVKls}6;7CZBn%lhXdfmg z2}9!d?qTwhFqAk~z&1>75{3$=m7@%kpM;^^?FV;;$x*_P<8ptPJS7Z$T6#SZCRYhV z;*}9$@|7^8-M$zmX9+_MhbTZ1CT|Ht@X@3&xl0%#CufAoU&2t$R!wY#$zj5fxP4KW zJSGf@U)O}mWy07dSD1Vz4DF7l9te}ugrU>NL8W2xnlPk%IKt#MVQ9B!Z9IZy;T{TtdlLLL;stzyU`%Z3poV{xB8!kyMIbozScJSPf`=s_i;!DIp~4g;$fD#|QONjCyC^wU z^yD~8_b7Q*6mndyh>~kXp-)Sz-cjO& zj);n>iIT&`Aj{DvMjjV~jJvgqk;}y(<89qz z&5 z$H)O=koK}QMjjZ0g1ssbh>;7%Ab7DPMm`wBNdXcgCyZgity+=A$O~hT_&ppWH;h5z zxNMC4Fa|vxweW`+IbsYFXB>@@C&r*@PrrX+o`45ei?_9k2!I2%s6CRY+;-{GY&mX zwQO~qTr&=lA6w((n{jA&+kbzYoHGtRjoG;*PTm=Z*cW@8+%pdC=Iow0`DYw@x@vhi zP7WG}#Hrahd1xH696!X#MdQ%Zd|i&l$w%W5e03sDPMUy})7A;{(gb82vu%RhGyxfp z?3N%uO+e(>c%KtO>|?$nXSt zYXTA%k4})gCZMMWySC}t1Cdg|OP~p5%WC?QH1S%vzN|4_spr^Ce*%Rcr3Fx$Of+s|^iaoLk^4$b<`ginTf}A%2i6f6B$a@nIJb5BP?wiE5638XVf0K~$(zZ!* z;3V|)Y0qv+^57(NI(DpAl3X|my&CQtlH|ilNF35PNlu)E*!}yGkB)M`DI?b6fElIwdgvgIMNpj{SWE^v0lDs(y z8P8jlBzI0i?8}xU`EwE~%vTenNpk2UB>pK*l1C>Y_`{wgmrg>K!;>VRPC~|)LP>J! zBxIZ+lO(TBLaX(y2b1L1Nl4s$BuRdqgy82BNpkEIM2@vik!Po%hHq6tO_6J-Amh+o zQ{>wzh@I<|BIiy)4bvzjnj-H`K~J*{?wca_PC;o`|s=&xp@kD+AR2Wiu^nU zHLRs9Zi*Z|1x;ILY)O%)r=X|FhV4s{tEV9GUU72_jWl{- zhB-wBG9EcB!@ME`8J8NBVQ!Iurt8M*8Ri!m+$;m(8Ri%nNL)5G!#pDcO_N`oonfw# zfll9V|2o5bBZD_&peVzfBLhuO&CMC+9U15}ym)Vhxkm;P&lhEwe`K&x25>XXK{C+k zbYFLdc}NB_{u#i`#KnWto-)6}xdvdmSokn(YTmibB+56XgMnX_b})6IVUv&>tv z(BB4WJ21=KB?}pE9hznSlEpOHAX(-xS?Fo}MXzU>$7CVnPw!=!%VZ&O=+rFpnJn_< zz+{=zWFgD(RhD^87BXJGGRxd13q75`cypHdO%^{&0m(AQ$wJ02in7deve4ho>0rw; z*U3Vw(;phL%y+W5R0?XAIZqaPT7PLe%e*HGv2)*NnfqiR<6eie%zv`b-|pyiJj)y? zOCMw~4>IV74CX=xeUZU@$e=$mm=hUzM+B(ByvRUj88A1P8yWOX2J<6>{>fmDWY9+$ z%##fIDTBF^L0@GsUoz;g4CYJ*wu_)Qm^Z2C7QbaMcQWX^4CYS;{g=TU%AgN3m`54( zV+M06gTBmQK4s9K8O*5+q{Sc^%&XLMi(fOCTN(6i2J@;HWzfeN%(D#oIfJ>D zL0@Mu-!ka$4CY)0I*Q>mn0Kk?mH^yf?q$&T8O*;7`agp?m_Z+CFb^~62My+827RHy ze9WLfG?&J(IrHWWHz8-m&c9Z#~ zN&jv#$295VP3D;<{k(c^3Gz((dXxF4Nq=uL=QQc_P3D~@CW_xTnR}Y_{U-BIlm6di z4r-DEn9M^>@&NVRl8c(;0w(iOlYGErPHK`9n9NH}>=s^NGB-8J4NT^zCi#KM9MvR8 zu$ZS>xwQ!+)VZ>r?Ymr-6%x^973yV3fMUG)H&$Y-i)N@O& zYmsYM%y%vF4U0LiMb2R{@3ruhbb5=quSM=*G5@v5KP=|J7CDH;JlG-+QO|vHaGqSm zVm@q$F>D=Ft{;jKy5qBA2n4Pg~?O z7ISKgoW^2aZ9#v#v`5xrZmphMfUCv)+9JQPm}6VyI2QA4i#*3-u5FR)Sj@LA@*Rsg zw?)p=!o0f$dir`&>lWtTEeHs}Zejl2LjKdj9K3}bsD*iW3wclrbMY2(p%&)jE#yNj d%*k8GiCUPKw~!aLFgI_(SrQ1e;QzMqhrQV=h?xKY diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_1/1 b/tests/v2/fixture/test_format_compatibility/array_10/compressor_1/1 deleted file mode 100644 index ee596083fb83fdab88d69f8caf6dc46b917e1af2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6176 zcma)9XIN9&+6`q00w!P?KmrjFLBUWg^iZOxh!PNN^eT#!&;?vOr3(imeZNg6Ly{f6%RhBrg*+sj7??KW<}&^^UkX|Nsd~zt8Od)l z&3Jek3Jn{W__;A>)#{6(%3KJYElii39cFa)COb0wCo0w@0lR{rH}|E;+4 zL5mNA>Zcm}-0=F{dD;WWjlQ)aEQ3)5j$?kkT@f6&qU23ljVM5@wpFWPWxVaizCjS! zvQFcG7r}#q5p`JWX{eO5=gd?M{Z`CoLv=_M=>vuu%|F0c=YLoJd84{oZj8^txwoGY zesSIwCf#4FD7gzY;56dpG@@cZzx-rrBowmQ`*zm=qsd1i=H3yvnYm!0!Tc&Ef%VzZ ziSQ{;7b6c`UG-7yQu|a{i)P-P6$1Q+>&$1lCH5c6qZT8$g?DR1aIlw5_v}dhxl)El zB1XS(uu8W)Wh-pF3>G#`#Lg%yAu7Lx7JI*- zhnx5K1V@usRq1nS2!zx1`V>tRGNvxuvU}XB)j@5zIJKZ^WW~eD>*AMr2*0qI_e~vI z=su>f*p%yoLgf6)l&{I#ThB{VE5G|IA*SwT?H_!;@WI>{yBQx%krv=LAmaLb+WvTF zT^v0N$1Z-GB%9c`UlkI^IFzW4J^z@chT9Yz)Ouq}y@L45{$*gX!0gSQG~dgv88=p$ zClCirHsDH#MfHH@R=BQL#@Em5H~3`=H#Kx<mz$Y@=~-BNRX04BwCh-{8xU&Em3-?da-%83gu z0W{yg+|`Jtc$vZeGsqb5Q^dl%@41U50gbb=;>XOZ=5~vuW}2!JAIs|M(1Q0lrm!gZ zS|~6Pm=pSHF)MSYl|&Rrre>%%j3QbUjaJ(aEwoUzUrP0g1DlbtB4L#aPO>Zh(3^#} z@cH+y_dWy<*=JitfU}d4aXuC*!XuXN-RO;~_Q_7(-E3s$!#zu_^be}`o+rI2wI?nm zU)TymcMktfenb#T?&!%85C^6^!MMgqt>r3V7s;UMCveYR|)L_Ptvg)Lh-?JDKAck6N zsB}sj>c$(vII#>aqtH?}G(NU$%1ih^#rcvV!}| z9XoR)b}{3fE8$6;#51O}7!BM+N`MNI-uN2MVqD5kd&2Kdv}4bIj@mPqag4BX0jap? zu%?)l4mEn73%N4q?NDsXRSLn=DK=R|9-9(V+8jy({!iuO>pOobwwtda9ZVX9Lo zBg?rEXfjhE(zpKQ@k*g@sP@iNdTSQO>7%HqtBLVYBTZ-Bh3Y)@9`DN1FVkN4C7@eF3~j zIQv!MCgg$mCdsBi8CU+l!pOF0E z*WDa6t0_o4JJ2z){3+Zm3UHp-*oDnC@mw9%fBEpdE&S$j^Yw95yN_{8c!bTE4f5p0 zv<9T1c-7-os-^Lpna}8a>$z#9;nU==6TvNWub|i@zbYtYfo)GwGE)>Tf7ceniNABV zR-dJ_BMx%#tdx%A{fHsb?%3V$^URNG?DuCe#Ks_zAY02qkZ#;wHkWZ4m*L@S@hIoK)c%gtc7+p)`;`TmycjBCL>0LHH=)Dl`J-$I%3@T zuc8KmS(n3?pzm)9@@kmB>;wE#7BpF|lTfT5a+B&sL9`M}!16hEs^kGR5sfqX(y}ay zi)3~IvfjX-3m6wa5Tz!PB~|1h8*tKcr$Dr@+bo?(I}YmYvc(dR{N1w|91QPA!Wyqk zo|drKxy^v_>#1;!S4|N#*0%0~^Or%C)_NA=n`x6{n5L^=@0|r5rN2zC+JE11k_jqp zzVO_N_UO$pJ0sOtNTY~w2hI`S9sgywh9FTC^lq>KWGu<)u;p7N0ics$SGr3c1{sOb%z?K*fvw7Bx*uIkerCqtava4p=`3#5CTu_fGEfR`ZYItA%e-b77D ztc7>oPDMM!!Y76&<4RTQUePv>Wp8&*QXEFqosJNOYZYy*?nSwj>q0WF zNOD;G@YTQa>zadh0hjk!h?5MqclYeBP)knAP@$;1LuW4HOeD)8%oThXaH2B> zywghbMlF}--hEm|bEz^Y;0@LOGwAfkB2mKSqA*6m-G{|r(I|AzG*5MY&wj12Y(*_m zosJS-aYwZfZ8>qSY7h0@)p_aR%Q$mRx~8udVJ&YXj#oEsL)3jdVAYG$L--~fI84m- zfsyw(eAnjNd_F}p%30vHlC;?Ako{t4f5X`KV+N;K9bez!lhkuKL(TEG&E_mM%285c zrKKqIbvBG^qpyr7wu-q_IPwNs=jgNFa5{VJ9Dd`oBC9RBm9w;BQdNPKnK!~G`OsLK znw`-c%4buu|1~Vo!7z<8tK*hfP=pQBhVi%u!Af=v=N$jCyHuIgG1cJ^L7DmW!a$}) zoJLxX8#`qohp#Q^j{g;QLDB%{S%xxAET~ko)#of#^4NUrdygtT_Sg-7zNr!ZKu>A~ zU)#p0{6W@MYYTRIXz(GzM6PRYU2`ZR=yI15N^F#5jy=O3oAQ9qogyr_7Pv8<75fv0PezbZHAAE@xVw8TyyJv z7!BObgF#L*ru=8|YKz+Jl%bb=?he-rRf9^CaeYzmPu($MH=2;B6w>Ue*)@HR@wno2 zP}Y#>#391Pl(%sjLUzG*mr#;5Fs?;T;@D|zyvYywpi+Zqg>&0?lRd}HSd1WKc-xpA zXVn(BM}a?;6Sh`%XuU>8D^hr$W>4HD|B!_F2#qq?2gSDqw1MBE#{rLzeGiS2)W_mD zVKQrCqlv!D)0{+Mh&Fy*5f*3(6AUI9eicZ3MXs}Wd^JL|E8mJIM<&TS7iWaa=<3F_ zTM^RMZEylGw^P=@tu#>LFzvo#h304ns!b^iRs>%qbLWVBbs!UqA} zR(&~p)Wn`F>o3%t8aGpmw$lZwo`Q1^5u^D!*OM-He!OsAz7sL4h ztEF~V0jo*?-6CN0TRr1I;X%NtYqS;BEI^EdHYqM=DBf418dmDA(i@{GWM>p}>SPFD z)U$C-&b~D}QnREI5mstYKm&}HiE_|N^u(PUC(_CI<|;W3+J=mk%cOJc?CvT7Vlb%z zy=2W?l1mj)Z@`-ti#3Q~gvTeNGTN-^GIBbTLx5NH5i**dD-omF9bboM#$V|6OBLX; z_Tr%%1k~YGd-#KIuL6r?ur=Y0Dbt0AnK>Azt6^NG8&q&xQi72xm43hTg%v9uP8P@W zWIfg>O7W05j9wz1#jz(gp4vOwaqs{;QBUu}`%_s?tVR=J6XN&^Gj$f)}DU2%~n=taV$-a#Ng``(N?hY19sMWF>K-%`yBTUYW&)jl!GJ+ zfD?)Z&s)JHH5;{31i(4~s3CCAFAVDd!S{bOJR$%T>@c?%zkvY3HkppgQ6*sBe;2>V z_9zJ505(~MJ8QP({G-fsfzVUD}x$G6SbN44|Y_6%l;ZDbJ0VEfqwZ*CDh+cZHn=A&DH4uSQF;q@N@EZ=Y?() z;ue>^Ltn}cDq#3l%O%Uyt(Lp1Hal8=k{(^md$5BZy$v{wal@Pk3|JTksQuf`C8g1y zz5tQG_!5dsu5bA#H1*A;?b!cs4D&B}x1J<#I0GWLMFE}X2Qp{l0bLpMo;BvbHYHC$ z5(UAO7&hC~vl&l-Cw!s{Ch4Oh+79-2sUc9_<*CZ?-%wyZsPPDc=jn5OKtb08%2N?{ zlYX*nm-Z;&ng9w8I^>$|FYIs1{_6<(Rxh#8*Y58k5Sq0LFrZ(btM)ha7v*8619Dp} zbu!jJ2X7trfmvXV3X@l_!pf#yNUFPZ38<>3B|XEw)F7c za&81j|5mn@MD6R7B0DDkOa|zfzXjlCZt;VE7P}SEPB2LYNTLv!@;Rx&@b?18Hrx>C zI;c^C!2=kGhdPDU^3Od0N5SG5|QTiE0SMx5{etQa#D(L zh5~lD1~o!4_!>pDC5#{oF~rL&{r5=#d@P1-{|-PP-$?}bKhXe&`fR)JA7~)Yptk^- zg*cA+oB20_M`D{9eX|7b^>gTU3CO1ZDzMhjn$iD_{WUJIXc-j@P_b<6r&Q1G5PTTJ zy-@6G2lTxkE06#8u)2K*0W|CCM9n9u9ae#V2x`oMDK01ehuuHMXTx>2x`?!*9RUCb uj@pXIg6n_uYz1x!oW{sf+Q#t}^RcRz3TPj;|NozvAN?Gg(9mn~x&Hw>mA;(- diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_2/.zarray b/tests/v2/fixture/test_format_compatibility/array_10/compressor_2/.zarray deleted file mode 100644 index 93d7fdf873..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_10/compressor_2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 2000 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "TL8NO01|6QG@70wAB)< z9->4dL@&ved;fs@KJ(1{aORowo-=c1<{XuW?xLc~R=g%wo}IF`002Dw`~P9(R>9%* zMBvHFzn=LP3LuIF_&`M5Kln5*P=GuDe8*p`D69ZPFcTA{e{&!J5QajE0z>5&)0F@K zDuS?0&FBGuJGCJTi-LiPlKT)YPXergx|XFuD$a3{mGpVK02i5_$pP*+rN>(=u?n-N zHuXVadHJj9PcKW=G^WE|=y5oLRXG5#fzb{CVJ{+yDxIh}r{+HK3ZmbYod_x{EC8?mB=Ny77ESk^rz)kU% zSD?cRi>~EhWGWqOUJ_Kk6RoU`FwhXyj?BP=mE0q-U_JIy1Cvj_CWr=id@^t(pC?IK znBHOH8Rn3V^Q5Miie|?QZD;OZQ3{0kvB&Iau;@Ok$rn$Q3rj_z4WT*`;Wl}y@ z8pQL`U0V(Z73BbW5>o(8*R@^)!!F_>^dUfk0=PBkh&-Z1;dI1Z0QVFEh2=_5*L=Mc z?Vf4~fOf)ApdG0j?4?sCs?*elcsv__BWQ;mSBOJhukwFj1*DLLuZ~!qkL1%EAqx63 zwR&n2cjlPciDHPX?rJ!~6(5~Vu(N&SbR7^K)Jk$k5c!63J+qeMJzr9OTh^nMzGaB` z@C`Q9keU3;C?w;Fd8TIFX0RN)g-hf3_?PcT=U0(&7-wj&)(0y`_!U0sqm!t2GX)WI zAkjgAEeShC9~R-sRA%U)n9qpSpDNX9F`{%T4Ay$xb(#QXg7BYi(;hN3Ie5T-p(RIQ z`b`LpJD@TuTheyJi)bN2A0?RL2}fO*7w$NTa{<%qGsHvTTW@)+6yYwx*Ww_xke}H^ z@KkT6SOu-39l88}Pa;HC1UZ%%UiHotB_A0hI7}yniics3b|`J9>Jb{zb9Xs9?rOV1 zghyg@;-Z!v99RwNG%84*2Mmc=lIM9qNNW)~)H#lcGkg+X45Ql~Sax-n5;%kV9K7Tb zy}QwmJRy>Q0nd0jSVaVV@;)=0IaKaa48f^!dzC2!yh*3WKQqM9|1hkF^;_D5N9p15 ziQWzHX9uh%1RUHp_ko_e^)O6%GM6OvSnWxI`wABdkK+sKSx93h-g=dpxWOmE+D+4s z=bOyV#F3@v-VA>eoVdRp6@|MqO*#-?x<%DhEp)_1NpbbiL1CVZ@Wev{648+CtijOi zc3|z@R7zf!guIMRmw0)DSoMhwR*T+oMLFh)GN_D=Fa#@8vVKr)E9=F%Z_FJ&qro^N zqRT!}^!*d$_F6Kp7i*V|bBEu%w5})fbn^Pim=^3~7CLZ`Ne`1XNk(5g@onQ6D1nC6 zY))aV#~}gz-_w&ve4g>Gvj|uQQZb5V(rYE*sV-L3 zoTr-qGkHN>V!5by(-#zyWsVmvf$4fSi8VwjtQD*b#Ic~EV?RVu<0a2Y6=CV-$(PKH zs9liE!k5&HqL6+hD`Iv{ItEg&bDv9@b@eu|qh1SuBfD`602q9HsS6_nP`BLjNw~PZv*? zz5mER>?~e_?^IuDx~wa;*UvX>&c2@+P_%fux6<%iaq*qA|649{DLh8B_{r1O0)kAy zmxh*)r%Y^)Rp#j#&!2W4KJM@#DE@?r5Sjq;n)9bYAFJ5sUj_&98%Eea`^B3_x`%85 zNqj=)S=D;psv$jn+-PPZDIiek=2W)!SKr6S9oe5JJ}zgO!ZtLHOZ&Lu$N6O5qb$#e z`)b@kbsdky^3AzO&l5Vq_fB->X}5>}OqaoVWD}x^cutr$ z0MJ2zV|7OTAEM_|{A#xL+x(hp-illzb$5e4eUzhJmj!5tMScd`PG;b~=59=vzJv}$ zGWJdBE(uO{TE9CV)ZFrlP?Y#xNVf8Mr78a=QW}hnn zQU$BbqoAO`-0-MWok7%mBcV+iG*CS^5Z2Jxwzu);&dfy;vv{EY4BPjW)vsLMonB!_ zyH_I=`<`J^053Mz27ROWs=b{cf1nV2=;zqvI_4_qt+hkemuJw$L-KBNg-Pq`K@A=V| z(|FKPe6SKqwNJte8~zkA6sc$Eou7OADlA+u#JT~#&%5>!+k7Up!UQ*;tUu z(2=Ahbs_-XYPxzec*+)O;WlKParC3_(UgAUt_AIjmS1njfBY5~=NvI3yj9a_wz022 ze=^|n-|3z-#H-n-H|Xw3=-G?xJv)z`%8Fxwtnzz2g05m~nS$l#OS5WLvF|*QnzGhz zjX<-{2N-`%g)R$?U)kb=0H#K7IW^t8!a9<#rl%|K6-K!|B?zlv&Ei!*&dgLnFDNMH zTSi0!&B$yeGeGlXveh;Vrmr%OmM9aH zcws0F%h%^f@@)g5cwC#$&u@!}I5{PonK_vSta~mwHUA7*<2oYL={wQJytH4~tkCpK zxRvZRVchRYzt^(rFUskY|8X;4la=Mmsolj7xWlS?DvT;+8@19m=+GTysB_O#(h8_LrWx~#%yfHIf&RE10 zdDDhlH~TD->1jyleqJ|HrA5mjG1ba=)4TIv|&qjJG}4c(QojsCy)QW@(He z@%rJcebiFqY=079_2o+|nf4euah!9%P2PmrYt8hA({#G=MJ}VMLGx33@JYDh z8OzTLPd*jl#lq0pm5;L)gsVIKCl-4?HWR55{!S}fo5;_V#TAQe@1}HYH}9lIO1<<_ z?==ZKg4Zku7nFO<_7>=g?fMkGA4rb-^2QGTAlym$L_Ft$q9v_9W~f&3s~qe3*;py+ z>V??wz4P&+!C0{I;G>?_Bzi}^A!e<(|H+~=U z(Q}IHUPq{QI#t78(9Br4Fu%68l^rQEK#fbwHq_&BHNp`W5PN(=Sh^cJkY(lL^2vv* z)U`P%WV6xMpy`Q6sV`O=V;?iQc@iPWHvd~jiAWb|*Z5s+KO(?{9a{{y}?Spr*8QKda-5K;Uv`O5d zIH9HqMuHy;y0tEeCCBXgSw`5alu~za^wWjD!fMB*4?G`x&F8Aj_QwN;#oe5fG!|QD zc8u?i8zxFrxo=DdX-jH-l)B29o8Lpz?H|%(@Er79UMsLGsLc~KBt^?2#xByq7XzE8 z7c%C3J`%l#B>3kh-BF6ooP$BX=CImrXoyIaFqdHws%Dfjhw)z#Kr8b14OWs{aD%=Z zA|wRwIFhWodKdV_4j}}a4yG*X=!uUSm2;;QE83aXz4MwjOWOt7L8-Mo&yZ zB{MamU6YMUGdVAegrhQK)1g7M+7h*9mMlemk7|c_WXS}bOvN@uX~@_|rK+xtR4S!# zF8TC)7K2tW%vMz8EW&9m9El7%aprs;@~dZr|duUS9ASx<<1 zTY6UF-66D=?|BO+u0Ksa-1Tzt_3q*I&fE zZz8UxO1<#-&jXbXPcAha6t?N8BX-Ra`EwaW`Djk~lW+KVCV7ajQ3$BEqMdwVKL+tI zO9vttNGKM&F(Yacs;xd+RYYg`ue&B4=g7X_#wyh?PMJP@OGz%ZQX1+i)U*KMNkSs3 zNt6$u5u!oJWafWPFm<8(-VbV@MDQ>tdhYi5&uQl|joL6EEnwLKc1?mMp~klFIV95; zR3xiG`6N!WT5hhEmeCVe0)k-fX%)x9`<~ojUr^C1Uso1FH7+J~FVo$Pm{E>D@p?x| zrNzm`^>~_t5l|By)H{Y+#e~|8WNt{V3Qjoin*dzcgNL-o-N*-BXeu}%q0`+&@a~#C zb9P_9FA!tVnQV&R6Q$Ws8$j#qy33-vF)=)F%usxf0i!pua_P${i;$w08_c>dA3#Xb zS;1bA!OTgl5uANh_-$f zF1*fz+;vVPr|4B#k1R}cgdlyeM*kv1HGpC)0>TqjhWWZW1k032bPE$CFRjfR5@wB$ zSxi{ZnF23|{s>suZoqAUW#%O6A|5;_)8HmPTSGcjNCWME*7-)KN%#|dXcd|68UWTM zux<}%nQ&0kVvp<<%qx$g43-h?)xCa!uV$SDBj#X9*8TAt9onxX;c>lHL)l zt;W`;(UPiO{&k%s8>Ydnx9D$90@mo#EMAk8u5fmen9QB5)pWpjQ0SE-dXfe{ILQw}gyL`?NjXGgB#7PQU>^d9{tHP~8tfaTAz|gvd0Y|Fo=W$+9zqfw zL(@I7T^<+nl-aoif2Fr@evPPPC@26!Md@0v!N;CqsY6tb6>sntW!wD zQc?2uh_!bMA}R?X!q6XZX~DF8vGN7tOV;NajG)C99B7QY?I`GvyN!Hrj_bfg_3-6! z`wYRbS$j7{{ia*PE3GSO`ib1-h}ccm`kXV`^7}{Hs#>`Uq6fPGoOn3I(AJTRgciH~waVBiQEnn*JV0s%pw+`;{pclK}wxoExcd{YlwJ>w+ zCazVBsYQ`d#&fB)@~a1hHv2(((ajZ6Q_F#HLXN1r^luIFQzmLC$6s8 z!=1_IeAlrmywApm`-caIvlFvTMA)8XJwy`y0HwW`=Hko)R-7$})@cS63(iP8_QH4g zYL|qgIOR!+LkkEfG05jdUTt8?RJu7)}=k@&opIVM8EsYkkC=`)4?XAA&7EDX6u#*W zzcIu?qGZdKw6Mm_4?xu!emGFeWHl8!iIg>4Tzze}bZY?lpZ3#VQ7Iiye~OXe2+ z{Q151{r5{sd=?jMeUuP@e%TwY!osh*wy?;M%WGXtP3aaTio&w8 z5&JbocJOOWO4I${X49V;aK6V=husA`(6#&b4<^-nms)xm9y{9eE<$ z11}#)GqXm30sA!nbVKQDE@MQ7aEmSuu&I`@?sFMOSniyB1?)Erix3aDz5NeDG4^RB zd{A1YQRpp;70@mJk5`(v{5QE%71$#cvPa*h+}~ID!Navf7T@W&;*f$O8qKY?%InX~ z>&bl~Kl;*3>ZMC0Gc%}>3ySEY$j_GHqHKP&~DOsWma!yD~!#`Qqa1AA|ev&mgY{vxYIC$wMRz(8=1LK zbtfo}mQILEzB8VzeH};6fu~U-pX7qoFmsDFxHh?Zx&CX|9-7z>(R4pHS292>WMDZp zH8oA}jcqj0CCF1#$V5?dZoIK78JzPX%s}?Z2kZ+{b@g|IUzftB??WPGrKgJ7nWDbm zU}tUB{`Kuw*h}8H1mnG%NdXzd!Y;z^uTSGqyyYt)&zbBEXf#bCeVgm)hZYQ`bp`db z4j_GYK-vYpL-V+DpIA*T_=8qbv?5<>{P%6DENrcFMxP~`GwY!^^MKg8O{{${C)3;+ z5!XuU@SRjN5_%S`NL|OJV6?BA0f2rhx~C>^aVgm6b4Q3Q*S+r-=tGd12sZR_GI285 zeB##%%)fYW5SV}YF8|8@TxNfKNaiZ==UAk@&8FheSm5dM{exSV+`FfLG$Nl0P3{e< zNFh~JX!>kjvTR<7W{Z~axFi_!$aiLCl<{wrDHs%jMnF_=k^tb=_$_cX+iuafE9lXH zGW6T!#N;*d^VD))Ev3C`QK3m>^XnYv=CO`^D<5=k#yH=&JvE~&FjK7Rln~3WZGkr* ze=aelH~PY>;Chf)_$%PkO#SQu^>9drpb8WTji$#9iYB@Iq$!t49shQxCSprdq&0Z6%4AzS{1XT7S!HDv$jPh*%`n#eqK}CMhs|xZH!JMaw{e9& zj}<*|*Twa%C;YfPAKZbD+l!XV2|-DMZcr8F`R?pE-5XS{eBv zmZKTyZ$T5YWpdM@ZSLQ?(A=e8G-dd;O`J+Md9H`Qg<-}SxGxVk3%2Y#Bmdf6m5ixl z)Z>$k>9x(WV}rifA?x1WWaW~(DhE#h00GzxRN?}DpRbND0i+KmdO(|Gm@CCJOoD{j z)FF=dGSu?WY0P)sweVJJ)K%!acMf4{1Mn z;KJYX(un`}6aA(*2+Jn}eWGl})g&k80n5xRPV%W^B@k*G6mZ_C}?5szQY3aB=QV$-zV zCbgk0(U!6E^LPFQp_lu0M|HaklLEb6eSNz9=X(U$oSxmxY>d1*+rl{#kR|^dXJCVu_Mjp79-+{cbO5fd6E_x1mB&VmYiM_SDh0Wc=Ii_Y+8) z=cMb=4=JP(p-?)bQuZ;YeQMmS&19rPFoF z?Y^?)A6X=6SF#?@@;D}JFZ?4m(%dgMNYVuhjT}+EKH1^S;<*T&MDM@T&JLrWw1exl zW)NiuPE4IK)WVaU>g(WiuZ9mBwz!oEI}cN;;q-8ZdZxi{vkF$z5n z5-(v9=839!Na7bhQE3@+EuM2Uok5Bo+@v6RHaj}zH-ozjl9J*wkeP~vHqJ50+J)8j zqEX=9s$J_lJ*FrcxBMcl>1B|eZ9O{sD3UMJ$!g0irbB9gFa$WZgOCn({)k<-2PE6^o`4YT2m^O@LvX!p7l(VF7k-J;yi(Y2|CZ$%gM zH?zHJE>eDTZg{7`bM5xppakSjZD$wNbgA;QFcVUJ56M5;FVks)Fxgoh_ zIp&s@X{KqZA+D)4jGCoRmYKFq)9bwN_g>#0-yh#|UFY2Aoa?#HdCvVjXFVLiu(EQZ zzeIE{9RQF3RKeKFOaa~zRA^0Ovh-x| zg|+jMU{c(X5Ji!ip)^Bx*)`M}#4VD+I|*FMxyobiV5t>*tmBx9z^R!91#7EwOweO` zU@(ZKjdC5$5@`cyhW7q%a8qxT7XV1LO^ic#(CU*Q3`65~z2gd5uM7q>g*EW|5#?eE9K0DlK^SI1t zT2Y+ghn!j6kIEA1U*80CwV{pER=&6D>SPnGKQktKR*%MCVP5F~hAs2b=hC|RbC*VAP9;Z*eqajnsYmA-#xr)m7qW~60bXA zMf?;CacK7RNPR`HZq?RJFiyVGCuKSjA=~O*aXVO)5c~e=x@@h{-kD#18NwErGW^7N8<=(enqTAQv&G?`CjUs{^Sp12S-GdFbqLhn$xf{Z?MOWGuDjYl+nZc?Bot4F zaEkQ9U6C`kDOUc|8rl_^O7z@Q?pV1Qsk`6mxV9=4pOn8<2|s|840k-uH%pLdL}M0a z0(Q1vm_$2NU8E@*@qSqAe#yAcR^tyYHi|~pl{(b^#<^w0ozaF$6u!Vm&3QgQlz+7B z!6QPuoXyij&$ND6XsI55%=*^cNmR7c0IAN;M!(>cdc!XOZg#OC%fA(MV`ZBNuSklX z-&N_$ZpNPrX!OdkxP3e30_l*8QjYK9$Ezbv<_@iWHqMuhdE#__owM4L>7LB9PK6HX zQY7?5i>VWv6aHeizz497m79C#zqHP{^_>1bY})B~P#qxexLM!+#xl<(Vz>OBy>vqb zw`TrvfI#YtZ6@560k^^Y(U&DJ@_i!rye*EENF`rm7zYLdvn{Q2my>g%&01d$h~57jYWV(928NWa zb2cT1S==KFViLANSpC?I%{8`3&YXdzZT-Nn+Ji;%IY@F$R-_(u$N zgGB}r8ISn2!8r%WONQ zzzv>pi)s<>uK_kE@@kgJHW8SqU-Qh`q`Py#ZA^r!`>WR3iL6j%y`slcMTg=L;i>9bLOo(Z1Wt_-R+FibrlIA%ZIn zw;Og0&kzLw-xXF$!1pl5!2(sS(R6lwOnK z4r!zl+Vf^n^3vo0LQ7Raw`^ov$(q#zBBX!HsRd@=c{E>+8J?Qn5!mt~aI+mV4IBl% zo#i9jqWbqluKd0$cw;ZCk8y5G9`~js3CKK zr#!G~*4QQ4PAY@$=sikAf z0ZJcLq!5}rrH?K0Y|U*dFOKjfqMm&v1QyYG|i4ijHH)6%0`?lqZZE zg}kuK>`~)RZKM2t&?19g*+~jKrmZdV^fFc@qM2aTWJ!dXs6lCvH8)AX8+Vs&?P7@Y z1Jle0(3l!EivH1?gIrJCjT4ZDdtwQf^t-o9^nqSA+mHCmcuwZ&J}ynNI;uP_S4Vzw zlAkSTz%m-wVWOZG!gbnnr`qVqhpYHT>K2iL=x$-FO_K)^l^CwBG@p5y^Zu$jbvPy3 zD)RpPVo$LQM!ku_I0Grirj7<3%&-P)_scdBLygNz(}vjU?QMHeL<`jtf)UGJ4dS4J z;rZM$fO6i**lrQ$NhhZZr$$qI0t%KR+i8h3`opMmd|)JLnrj@C?Wd`&{~iJEiz=W= zxHfuIu6>pk?SE)zG3bMCkdwcSZEI)Kk fXpXzs;g z2u1pLoB6^@Hu$*uq-#xI7!ul*Hp%sYLWVqlrFHu_{dSD()|ECSkSnimDy$j?;#c~j zVLg?BkL$tdm@A>h1#E6z3m;ASki`<5RF9JJJ)3%GVff<9W`Gd21mAmEkm^-(7T2@?+pgP z3&R4Jv_?z?N^Nc;RXPVeJgbJ%Yz(>6Uav@+fHRU#!F($UIcxlpC&)X=s3WaxBIN@9 zbjVG}c|-N~);XxK3gV#3FH(4KTxY~{Cn4i(GF{PbM#JW;E9ho6es5p9ZSs&f5iLDW zVAqfajmOYI)IrParBSroVX*3l*4e9tjYSF#&gK(T>Llx&p+>A*&F~H}vO*!Sq+61y zIAPpm)YnrPkVT!FCj_9_a@}f7+5tnkn$}n%uB-q%9;h6$PjfB=n$1S3VTcsJBs$d{ zmRB{@*OgcK8lQI^#Te-9PNg1q#%n>^#fBNfqx53jA%bqJf+s70hGa)~|S}NH zmOwNf^SegvG^z!u?U{{EZZX7Sq}F7cA{vnrw)*n+=JJP<7kr#bX>8_V*t>QE?R{;L z9^Il56+(DwQkZliz;|!NY(8j6xi%1T&t(eDYuZG9*QNYY*!$(lNiJn>Lh*sdez~8c zYvYSYxW|5NU+aFMzUc18T+NVd{>Tb6-M0*@4?KPQ5;pMFt=ktbi=1o5-P|9n*Q&^v zDl2RBDC~bofaP`DIXWhM`EpD%75_(Ti?pn!d_o${n<}cDbyzLCaNcB8u%_kuOPTJB zdDE6z*D@LA%4^tXEyQGJ4(1S`xz-f(^b>UVLcb=?m+()0wwFrk*tySY)!%g7*chgq zH#VO1xPR)}9EE}@wu`pPM(v&)?{Wzt_mu1kyab4^vf}g8SXXh;y(bL}D>Zcfie;%N z*+>#cRemaaTXu>2P3vCb{_nI;_umaX_)BuNVnCEUXlimXlB|G(i1if|GCP;M#@E8-^Lbi&v#-$^6bq1FR$0DNk%P^TtOX9}UCU~o* zkB=IK_Jqs1*_d8MeG0<$>u#Mp%DB2h596bTva{bP;bXW%LmRpu3p;A~Qwz_opGBeU zUn4K}S|8*#6X{_Onb&OM*w%J`c*PeFq&4?lKXle2%3U^##|+H@hJpfWigns2`_wNK zEYvpAE5j|YMAY{4>;(3(b~*4A#MReJ%lz>1-R5jY$=hAkG3O6XpFebD^?cCX5pl=v zF?Px8$gf%LG2L?NS|0noR!;-q6Td(DbR5#pm{QvM`At8%JWdbLZw924oOTMPoAT35 zgzpBV&5pZcqFN$wIGm4aUR3Y#dpdsoON(h~>AcGhvd%&2Uq7wAyZ*6DDP?*7tFGMU z2^rhnYyL2tv;811TgAi6#Lc|vsX~aQMm~Mpy~>l1yIgcsd$O$rN5kdQUV&Bv8Z(PG z|IGWlu^Z=NvJ=h9+>;W`N=xG|$aR-~8GN6h^Ql%tb07QVzUvF}_MiRlzhg^2b)D%y zJfcc~@f0|sNM<%BcQ1$@TkRyG(_Nh9&p}I%j*kyLvka%#g;{1dMEuY|=PlnTP zxFnY6=BA~kmDb^$4T7sYNJCS=Sro?1UoqLLg^nG0^U*P$+QncM&B4^H&XL_V+;n3N=z6xcV1}K z>{zLLqScE%6v4(Vu=oOH_=)6oJtB{mp%^5>J9uZRuMlSx7Rm9Iq}M>bn=dsVti*xz zQXsYCJpQ1CENz`SYL+Pb>->^Zr)C-(^on3sk9813zlheEl!>~ViJ^C@Ag9-pYmKxd gA7}}ch;iQi70=6R4tlB&K6mmE50g_6MVQ8a0a+e5D*ylh diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_3/.zarray b/tests/v2/fixture/test_format_compatibility/array_10/compressor_3/.zarray deleted file mode 100644 index de26b1c01a..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_10/compressor_3/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 2000 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "2@e>!n85+yz(r36eCfs3{ZT3x13rZw|935Hxt{ zMw>ZN2T*iT>^=`>alAfh7L(TCu`H&N;n7lmVXB)}JkYm-QRl@t@^fRIjme)`RTm*T zV(B9X4uWhU9=Zc-B}T58OLWA!1od*ZUSz5xyO&rqW0FhrtM49SVkC-hNfZ~M9p3^h z6X|FyKLk2I9^&dJ!!w9P9ij?U4KGc#+l}Zy~U`**HDoP@nApef!XtrN% z{HI#-xvQ&FqL{^jN`NU9ODl@LW>!t| z@c8x(0YF4VOYn8rnDsMbgpF;aIH-n8~4KdBcI*8D`rO((^T8~m1% z)-o5Lq_ZF+Dl&|~-`dmv?OVi`2lCnBr}aVx$_l2C7k`RI#VIKZNDYXL4b~hTg*P?8 z^bV5~LMv+81f&(7nmHuqY2^$q>owh3UYZUB1Rp~A0Dl%1$_R;Tw2EX1OM3fvdX1`7beIw>=R`BUiV=*aU+ zE@J2z<#UaLJL;n-4}5f*;I|S8Cxs>l_-fcyk=EZb>}=ZE+36qEGOh4(PH{vehvDaD ze%ADxDJ9NE?*a@#R2kb?5<+$-(NA`?WLu-=8xzhD`Z?uQ!b}6 z^FE{6v=-2dxQLUF=;q{dWMh+sSBdDUI5f~tG?*4h*MI;0JC+0~ig2lanVN4H!OhzS z_-Bq{e-8O`gA29KNJXbyNF;D;Yj#@1*Jg2TS@Q)~8kO>@JS-U-sMMYbLHufY2TU=5 ztr2!z>uD*=QQ|zJGGs$K2}{&3`dH4FEHd;~ebAnfWDcE#fIwq%Qeyi=Ehc>Ix}ttu z7d#nqatneCDn3DUWPRlIm{ge>JhY52BgG_kc)^oQwG!`3*$8h-Rf!C7!g@oq)KhJ@ ze=FA7j58})gApU*UM(^z)t*|-l$HgkB$KS5%EQ%X=%w{8?LLBe86SVA%zhN)r~J#P z40Lu_2M^PTOPp3$OMP!P-V?YpD$%s5C}R5&RPtIm{UU5;eVT%wL-MltU^-gRT5+w) z<)`^7=~SoSNK6334iM=a5wAb3ubH79aV3tOT}1{ z!MayrQd9*PgckANG1LImgkKOcQ0s$L;O6E*^CS(+zQ2GKQyAYc9js*mzY%EJ#Kd~3 z1Z+`VpThTDMKUHai$)for+mcOPZX!AtE~tW^SJXb8eTrH!p6jI>b__J?ZBg9q?krHLim#>=2%Yf#OQMtL`Ljo&*7seO=RH( z9CWEpd(H%EU2oitnW8XwA35REqbwTv$VJe*JKj#t$3}GM1WOOdrT)phU4(LLa)7GI*`RRKl3z ztjxen6H8Kx{1v4Do-!z^%+NNk`1q9`z%)6OqCy)cm5-#brNXq5GKiDN$cW9EbswgFG4F85F6#kkqsKj${50RP0za-`G^+qXGLb##f6GbW4mL2a`906V> zy5kA2$$(;ft*`c8+eA|(V520P(uFR^_Yv`uT9IHQ(_ge3Z(dlZXeJ_MA|dgTPFQf^ z=~=0&e8uG@UHcJ+jE} zL8<+}9d(gdi9TS^7Lx|B61|HN+knI3h+|)BG4Wu$S7B71j)v;ei6j(%5aSTd}xL2=!w-ew()E%{r5h!db#dzIe0X3*qg_@lt0I;f3G=TCo})lDf}CPcYlptXJ9%3-Rz>JH*)@H z_U++1ZQ~<;&1m%B5bwvkSGUK$ERfcxSiS2e&<}C?{BV7DvRd7p`VCFJ@#s9I6~?-{ zlM5(>J*{+|6L6V#PgR;sAhrc@;uz?FV|`+j)``Dq=a#HhXt(mgtto_s1V^~*aE zR%(gFGC%Huke7Fy5r$Bn^7Z&qC51sL&Y`(HvadH#T20XpXF>Y- z5-a$*LQDlpGap~}V5njHa`&G8ZD1$O&lI@t&Bl1bPxpe$Ct;(dJp#u{N<3D)Vufh6057)gjmLq;}ol_^x;6D zk%f^1@(B6oH~8%kH>4=!pdz@Ia`X-W05m7yfB(`$qE!i0Vgu*4hjYu&n3qjWFPjwzHR)Ql%7}u>o^jdILWN_AnOSkOY*sNmbfq{;O` zBxJbLb?8@PSCcDsE;TAoWYgiwlFxzb-EY^W$i9!+CFicgeb2aT{C3`Ge%{D@4xd|8 z4OrqiX*}Tm-1Kc}$#8JVXJCo+w$bjik!GTi;+K8Ln~Mx-uM8z%#)G0<43*BvSnZ_= zDb(c3H{&7SZfmUe*u>|6>+Uz#5-_6-kkR0sVdtHZAUT{NKCCA`%wfVm!JOq_LhoQg z?XWfR_v&%KS&Lz3db+^YA?=y?Juf+Cmdf3)lk%9qS{IRb!v`7E@bKPKbS^B0Pu9a~KS!axVamy}0|8e+z(tCAHQ|k|;63$?`27xeaU1-WMI~2+j}QVjg@E5c!0G|- zap|&~!6MaQX$V*d0!B*C4lmaMRe?*Yz)@B2QPA<$P_4{0ZKO5Zq%ecR~rL zLivT|-r7_3ASW=$N}FCAg@~5C-&$4xsG2;9Kj`y zsU^QPOUCCu8Y$~uv$Y=7{iuZ@DNo=kPqdU&Eb6AaL?OTQFr!vtq){TKQ9^V=LQ)}4 zF8Q5bQjuHY__>5FlOtCr648UC!HY!Nn?%WzWK<$G|S#A+jBnd;ZWWgf9{92Ux*X5iqKPZwLYqnYZWmiD$I{#|C2CtJ0uL^ug! zI)A*fFSly@*hSGye+IQ|o@@!cD&=%2g*WTn0?W=i^oF+7$`(JXsTy%0i$8ytkAGGZ zHR5Q)L1bS4_?h#wj}b>5#Nbbh-q4zwT^eiJm(b?7;Q0cLPHmRR!rvIO6aE4%SGIbS zX{s99EUQx42jOhlE|ak%5%2D7C!_IcWrA+}1jOYsQH9+swOPcaV0CL~ec@OL1)2ZA zcisPA?4F;!l;S<^Jihyf6UTSg!K?rBaO>Lj`d>#U{X?faMVGs5*MFr?C%B#c*ZVZ- zPVoO+xxMa$4%GK1R)3#ljVDjCr=9LnBmc^C|1Sea|6?5eJ3id_*WmvBY#hS>tMhL8 zw$=CzxK!Yk$-Hav>%Y^_Tjz=YyV_DCS+naL(EMLE#Q(?W2>NHjb?<*YPX?gc8LxUj z2e#h|`46|xTSGekF`xU)b_jRJKSQ_EC&?mopK|mct<&O+6NbfjnbuUk))dI;a?Jda zDRip-6mSAK7JBf>c?9Gn0&?5{IXr+Idr3{0xMqZyX0*8Gl(?p{xF)N(CbGCDnYiYF zm?lW<+4Zj3W32}j8{~uzA`bGo^yj%0pWe~(bnzmoFFb}7?p&S+-AhFT7p%&_vSncD zGB6t%7?TVvSO(@T0~3*fS;@dmWMFzSFx7ezs}#VcKiZl2zBgjw4tb%&i^3g(!X0em z-%(O#K?O8HA+3MTP4-|QkZj?3AxyS#K(??$w$S~94D2x|?&0?S$@`-aCi6jN=l*v2 z&pG=x%ppUzQ13B_<@OwN69!6@EflaFOr}@pFVueVK}O;JmilaGs=M&4Y8cFu+sYN# z$_2~Zx~vUq=ep(@_Q>3NnQ8ZZ_-xK@Avs_N65wTG?_pBzZSt#?E52n}Ny{Rm%q%0# zEF;3~%F|@P?sm}5q|Z*g*G_!c?zVgDW@cl4dSiZS!!LwdHZdj1MlZ>xdEk5c!1wu$ zn|;rq-<}6Mo*Fxz?U!Pt=VD`L#`~w~1N=xy2~v1xVzZ}W+h<~ir(&e1V*Mv#A*bXa zKyncv`4xb?T%B(wR2)by6Ey$RYTlPTiV>1hJxvn@QyT(H6^Y{*N#ICI;Lu9o+==5% ziQ~A_L=8|!HBd(_;6dNvL0{oPCGemCJSZm~lmQP)jt9lZ>#Bb21U+`*-3RY>9k2TC zAMfAY^*;n7J_K_<1k*eO<39vH+z0R92hZIH4}78!v>(aps?O*-PVeF^=&DZcI!^4W zo?G?pTQ%xkC4;V#^}G`eZ#$}NgXFgH1kZKsv+y?Wg+`2a@fkR+Q?oVa(p z6Ww~(f8eqRSbaM030gXJMW32*0u7%I<3UfYK;?lx`LcEmCado~W?Y3_rg!WCQ|Gon zN_VW`U(PKDi>~!%q?JdNcJgNBXY=~y2c(+5G*N3Pf$}mKCOZ`P6<)sb_6f?vEI92T zGCh?dB0p^a zUr_T9&!(F~olf0ol}^>DGJVNtZq2BDT5?)K{7F16T5$w!U`^5ek=Y#O#anlrRJUnT z+bU^PdoU2wr}6`ab%iJTo>m)0sNms?3QlRpA1M%AIjR-RP~rzvI_wVQP~?X@+B@np ztR0H|Xkf?&%u_~bROIjOUm=Ivh7qBAn(fxRUt732BEwFXbw9=bP}_Chx_D`IvfP#$ zk(r<;MMwqiyp+vh@OE*AQMP@b<3F0cCQYjJjkDQRcfkBC)N5E`8_f@O*=p;N_2TPU zzFCILcxs&R=8@i@y+m)md%SCd?`?~IliXJy=-60V!dqH$n!5nodHWyN`1?7`0ey*) zR}UWFxC8Gh8~`o5OUpTcyk&C$FN8t}@8~G|`fBq{!IJU$*TW=#o7~1c_D%Nd%1dTv zr@nFDa2;WE-zLt)i01fr z(l58B22fx*FtcrV5t?rm*>(si=WnZ9OcVOb@##u=#^p*=DQ%^+zi(FQ!E(6hXNQ;e z&dqw>V`a23x6PqV&bRH1*2?#4J9?^90@Ko^9aD!DLfHHz9P*w@PNi0g4HGTJ;<^Rn z-*reYT=xnwDfTk&{CD?mr>RyPq-nLwY~S{=b*Jx2@shiV?;jqbn-NNF2l_n@5rLb%p3&aUpkuX}kW@3>*9N-lhqR3fZn3 zB%&H6I%pY$_7EIKqR)q4@{sBrRhT;7jOcFaHFES0wQ4ZDu!{F1vW}RSp9{i=Q8xnW zqO??Kq3Xazc@7F~t3$og%o@{%4;Tps-y`S^vb!0Uzrrmsm20q3sEn%ui3ed=WHpgP z$T50DcbPdox7e?O9t6^;L_%iI+zNR*Hq_&yy!4YwJb> zzb&Y5zlew@8#Ir&*3hf>wILOPnVZ`?XDapgLr5k~ef6Wt@p^;R$=D)_ktK~V!yNFM z!#@jJAE%g-)$zXNnOKPxq`B&0R}%8b|JZ!#dIsL+PSG$E{me60Bb)T&dUt%T+mcBCx%7Kx6X#D{wg%$}pFN8Z1A=RN@=w z*{$-jtBsPS+P>p8W=WcJCwqrQuR#Y$cWNnWUh>#-TX8Yb4Q}8fq2kW$)y@@;A4Y=Q@1=y?- zb6A-}FI(h2`$OmTVr6Me3q29+8Yy)RExA0SIn>SdwJWbr{8P>*&he_ z$^h-^Em8Me9co&MvT7fFR+;$lol{xbnp6tyDVTD7YLw-Va!aPjzv;3c(3bEDHaV`t z_0vfh)R~GuDF_)H(zG@SOyE3VjmpI&5pbir5rlonD&k-v!vGJWSLD#X(Wf+&!z@qt;fwa`__$EyKp8#*>CO zrE3Oofe-AB00*`pR;`c|jUu==bE&|#N)4eoS0To5Tn7OjilN3e{d*}Gl;xxvYO1JD zqmkEy%VzsW2*+$j!0gQ^pGI>xk0og*wc86Ov|Y9u%r=8U6yAV#4BB>S3?!mo5p1#} zh!znHoYH93)xW|U1?coW)@DqQh^eE0Y|8l33CczKaq5J5CL*{Zu>Qct!7g%gK$g2B z>@uTLw0Zz8twP660y((IG!o_=yp*w(&#UAOIT;pUc*4$ zeq^qM?X8@Q;VgPKF(5-SF`F&tzjqaMTV^^9E$&hY+F>V@@;+79Y_TtfB(yh37aN0O zjf~+vC2a2SllJ}2KuqalmmK>Q1Yfu1_e*pUSt8+V(x}{h1{708^B)3CrSI!7Ta?i}p@XN@(I9z>}KSoj{y|}C0 zVKn5=#w8=34R1-e{DB&|R1Guw{dfNbYR;+5Me5;Wtn3PXzVQ+DA91tgZ;+Oe*{`#o zDw8r|W^yC-@`oO+wZ8yrSUo(eg^kMY=i=fc1|)RCngYsyeu1m6s9h?|SV~KddZlNl zk`Jiq*SVGA&@8d(q+~gA)f@AASIg#P9qJQw73>T@g{a-mM&3-RYrjzJa!7V?7E)zM zrtWr*UA}_ZmO)I?nI?scm*(Wq9VvT+neM_%<+qNB4EF=VzsP+78{A_gc7#E(l8VV~ zozT4E)HVxx(i18xtvBw;%;&5(XNcG%$Rw429)I5_w%`7#AQJ987HW}B|5|7vQ`IO% z{VR>_=*XTmT&#*Xwq^nm1m6yzpWjrQ_K1(~Ug+?2EA-i1lxS=aHqzJrrq0I0J@}xo zArP9sn>}Ou=O@6HF`$cI;=2lQXUz**%K5Hp81_$>eFk{pr>fpw*UvIzPF5z>G4{p> zYO|{&KflKHuPFB^4(95C_Hw7m?Bv5Ea-<3Mth8Q@HI~bkYgoHiW^RMhEAE(89r|#G zXv8UE1$$m7^~5rM!No1}BM!g@hpwU3-(j9-?;|=vPJ4aaBrG&dw8CjU#8yX%BJxJf zaf^Fs66&(Eak#e!|A=HYkmX}nZ()Wf+PQ9(%MXs0ZF*NKtTD9aZ)(h|LFyY);IFkT zycxXdW$a?miJTCwC(a27Quaf`7sc-V=UfSxgD86h_;BREdXR}=a~7GS()KYstSV+S zvYblxk)ERIs&5C5d)EkmgWetRB^#jowga=UR%`4fbyc5R*a>4)A3TFObe~HSAKxuPs_yW{u|hg?Gg0a!s0T zoiuctq5D!U3*;=tmNMj5Om=z90QOjyaU4GGY+m?LAJ&?8ajw;@{J>eGp8hF$cm`1vk#DU~b#{-MW2v5H z3t?8pkS$L-Ug-hj&a2tV0sh8rCyCe*fggPN=0T)2YNjYO%uTU5ipD6ErSs5HMCW?N z6Pq90KlQVi;q+jd1DtuR8A2E21C$vg5d<}sMVJU`mbNLa2%6BZS{mtm(~!^;O1Czu z>`_SPfGee|>+6p-tgEC+j>hOVR|4MJ&SVs;{aU6S`gLW~bVdpw$%Jh9=Yv7@UO8`Cpm2`m(6sV}7ZeNY4=DQz(V^>n9MDFt%+ zDYie^4O3$x(>Myg2J(?};NwhG)}Z=IT!Xrd*c4oLJ|3`iI(`MraWs(#T4I*u{Pru6 z$W@J1JH9lcwqr_sGUs5vMJbsezcnKUZ^)NNu#_`32Rzj8A2+VcxVwzQS(lX+;mAB?CY*U^W}dT0vcf4VIeW{g%v=cBm65$A95T|+ zDzdUt3H_b_^LoCo_h-DH&-eR!KF{m(e7-c)sdN!HsHi9pl8uTAN<~E_bcUi0QYQ`( zs*|Nu@nY&Xt}wyw6kZp{G=4VSmi+Z)qc+i1g09pwpG}>vTJ&EOnC6W@8b9?NfB>L^ z_NB`bc$n76w6&X$Qy8hOy(_quoxJ+eP2ll6{{2lHi7l3zCHRZ&t06?ilq?$lR4nCz z9RuGeoo0l{m0MSENWs0#gRm9d!3y5YyyS1C)Y7M+h+J*afnx8E+B$|3%6?065j9pr zjjAE=S!@hI1o_d(Nc}D(4x743FG|D4RBF9*KzXG!BF`!>MA&&Q&~tQuBm*MOVG7JS zd`1?u5W+(9e1q>1Nv?h5sw!?q7%tB%PX@c6I7Xh@PrPoo5S0265!_B!b~(?9CqfVq zw@vR7%B)`8)u{JT@>zuUB4Cpk)0@*Q_Bl^94->CdR9L>vSwhVn(76&a<;-b*mA2G1 zV+R#|XE0BHlgbi%v7L25K`&9SKbT92Pe&GOofp87m^G@oCZ?61i;y-~jV$$*UMK!K za9n3;)MU#;2Jb9%MEBKE*$x~{AdGm7W)Gq!)~vd$0E{|5s_u#Jp4M*NaKhkCMVO3e zHl?3p;*qv;5z9r?&sG_b&Cr4cGOXbaU zrE6mB)msZ;P?N}{>ef(IaVXpPeR-@ZaLOOwmO{I2q{e48E&ItS}ZBQX0(%op1M#`-KJf{-aKUb#t zUe$z%^D5qq6lk|7xE*$=+`zyJ8YpcwQ15hJnDUZBSL;|Hm8P39&fBJ0 z?^N~3Q6?8{4~5FaXq~bOTXp83-OS4#A5AXn*2sn7lY|$HBN)5B2i^kIP4PO(#T zzA8_))@Fyc$H!ZY=FMLe8LOZ#d`tLx7Roy?(S<i4!Rp z11_qbTm_96It4o}x({q%8-Y^(t!VQ&yl{~JaV^0R%D7>;tFR2ur z1Iz*mENTEQRNsx(?sjwEu7C+4`dxSW3QTSSlei9K)*PyP7m_f%)uVx&6^mUHSG;0@ zw%`jqVKUR5!?Bhx;aai;tnqea4#B*tgPLWh*vp`ygIVxaB1{I$G*Ce0A6T$G%(=oh zWbL2kZXBIP>nA5DAKYO{uNwUxY(7e(Z$T{qyaLjwE(Z3Pto&_4X0t=tTxKt znj9a4{bj$+Mwy1UpN3tC?_SlX3U9eKy%5;ZYhQI!E4=#DWvf6=5_S^(^Wuup2mA7y z=m}}CnA)eyVk$^F6S`C2@4uP-NwETYr^=Cp>MPvO+nUcu8}O8pt2ca6H}$j@M%BMA zNsa%PfAhjA>_lu`zK@!P>D1?#_voywCr>Z}P65wTAAg9n43D@yv#5Uicz^%RMUFB% zY-$_QAl1^+;Y0rOo63%RsKLe!Bg5~GUu^$I@4s0|Im4+PH`u&b+drsIZ z&?Lhzk=`s;CvUss$UhqbAY%^4h*QH8$3Jzo!g^p==2Mbaa}dMg?}R-I9rd#WuDu?h zsm+z(+qgS$IrgTY5@^7iWl!f(B-Hgq^&i%~zKxH499}oxJN%v7wTPgWcsPUIb6tJi zdpQX*Cr+<&$U#dMkT&{!nL1q3aw~Lsc<{e?K07-$hJ#p|={Epn!;^s%-QV14y&MQZ zrBk(I`gW1ePYmKum2U@AZr7XOaF`xlb6R436*xfW8d* z)>D|!)kdl~uW-iGB`;cNNG?(^)kzFY9hCQ9UVq-KV)dUI!H)e|a&7{u18y4v-w_Iz zXf6Uz`X4*rch9I7c`guXNZgTF_>)Q5VFz~A!mYGAD%#4Ibs}}Q@>$;P?qKBGnVe}s zA#l8DDCAWZHI({&UPmgS{2OYHO!;ikfa-Ikf6{%wr2Bawq-Tumg&w-vmrrGOIs|#nEkl~!U|Oj^GNf#*Kg$}@qXH= zW)hpM&7I(%fM9o&XtKmb>O>bSc_fIl8%gw93Dt;VcQq@}vb&9Ij`zB-kF^;SpzMMY zBa+IN0Ee+jxzEtg6MX@5&^c(7Si|?{vt>C1_A0s+)XOds8OIk2U7VAfJ(#Kg7%&5! znV7Ec#Y{t|>p5QBvP4>tZP7O5JIbBirH7eQm?^nW^^=%Mxq*O@fC;&Yi42nLRB+@| zzg&-8ESG6S7488ki)RqRsImdJyf#9>;$Y1(2#qXp%9>2=!=Zl!}NT zYUILe%&a)8J*{!dxN_W4eP4Z~J4BPxWjy_oGh$ngxL-DS@DbX3Fq7FMbqCd`mQ$XS zmm{2;-akdDU1(-9n9PTUkZ+^y&5wNO z;{mF7F$5KrrrhpxQdaR~l?ETmb2%gtaIa9^Uue9BfCbl2K%*$2#YNMB?cd_8^N{X{3 zXzdN!m$N zu?mn?i0ed|T(MzE7c5o^awp8POKmhxlkNVc%!Ob-f&h@D?U2`H6gvpf=7~}~@aLFc`QwrOexRWz*ev|*RXuB zcQu!QkPV>#R>55#EeN9iwyp+#TxmS1TkYlR+gngC{vZc4iump2JGoHnO*t39jGtxB z5mqpyJT=o{tq~UDWEGO#8G{vJC6gUs0 zy?1Y7uvTK|Y-&Amw#s%M*r=7bcZOEk=GQ1z(=lfyJBnHoM^S5A{H%(a&iSF|EWamT zpPxHWf#)G_~T6Q{C4L5 zWq0(&=f5-INiT(=y59;4qgd}!Z1=bE;0F}VhNAtl6FY+|1DY8^y6I{&Xe;+ED^Aed1cBSmT?{Lmj=PfL|3QR3(H08x(GXIhF$QRdQ8` zDu9ixjfD-`=DLl(jjoNR%}pB>8wDE~8%Y}xn*`g-Hg{||Y*>+x@yCbLTPAaYjXItw z6Yzn)qu*zv-ARtQ>vLk{!q|*cosM%#Kb$*Q=n%HK`$_P%j%~_YxVP$fXyVV`>uHl! z3L_QWp%^pDo%y?ev1UT>sg7|<9$Y~+JGkY*^1I2n;0qm{lq5KZs&FXsCv5B9s8d1T z&6F_sPvniMkZ1d_)v;tQsLlf+)y9zSL&CSQ;pLn@=@dKoJETiU%iis;CWFQGIwC13 zcm@(4@&aFxq6QCN*;V|`+*h2`_wEAoinYKm(Pie|;+Hz?DPnL(WOlI7E^NtU)+x3R zoZG_1MbBNvf+ z5D8iAB9dQvF3?2pm$Il$C89J63sGVy{v>J^5Yr>IB}?IB7Ez_+(|`Z+sXqYREI9Ok zTvbSmKbbom|1t7<&-3Rqw-48z+i%^qeXr1(JnV#SepA2YQxuYQ|3jO{7SY1UkM(tS zrLg?35IvEg{P_Hd`y_?t(EBHKW^?uj4-l3AZJ93#{A-$MI+-gAH&8&&G?;B&TQqnT zKKL(aE5~lNk)w6Q>nhuy?>%?zo(_G%>=!+0I%#WaHv4Pe!lm#Lk6 zVegy3zX_NMkFmD7piX?&cH-1(He+S47i(J9@2RhN@b1s*i~E;J zTtnQU*WoV#V6}*X#6}EhtkIZLCr90O*38Vr`+AQ-8l%P}t)9~7H&~kJY{ol-@hlob znySo=k41&L`3sK1_gfd4hkz4Rl~H`>mj6}O$49J8AyOq}E6BUc{27BC*5CFR$FLRfJw0=1BYAK;t~`o#l^7K-Bda+upEAo_=9_6PKzNUOGI*OO zG!4eq2_;|$eb<+k8hiE3dzpn>5zXaLlL!5h1q)GYnwQ|$+OX#Ji6z`jOH+<_foPVr z>+WmT3rF@V_L_*8puKVD;NO$2{nxU~6NSZzT0Km%`vXZSvy5fRdfHk{_AaG}cl~a9 z0WnB6f)Vk~Jo~-nYL84u$Jy`WDdkZ9mE0nej;OU47p1P*V-FiZw-ng1V~+noc3S0y zg7oy4N>B5L+QzO4+7e=2=PNcR$~7nJub|6eDzYt~D`(TjL7D4?6E^_6gTokMC})xu zrvEQpxA7C5&pH)v5qykwCNT+vFU1xagb5EreOLFcx~LCHH#zddc@IYxv>$OPr>9m& ztpBq*1sgRBZ{wPDX1Z^Wmo(&VPwW*lW0D>@(U%*h);^M7;uLv`q9Qqt1UBy0+Hz!Kd zJq2zH_asWaiT7~T*Ct?Iq7Yf`iruO07Ps}}h^2U3mIdj4l&upnkZH_;5v%QE2}3as~WYv0Of*5%q5243XsTKTkMz%MigdC*pmHCf}@ zG)(1`GQa0H?M9hZp)PjvVE*ul3uxE#$(>Hxka}>&n`4!9%o|L+Kw5!u`pnbI1S^jx zx*VNd2|l4S?xidqMK}%RNBy8Gznf*nwaRLUIG5WVLxkb0Xp=oxibj zbZI|yMx0Ky$KaO~pu9`?pN$p{On<5W=fCoK=%qQFMZsVNrwPKy+DZh*9hBPY^9D?4f>lYQOc(_hqb)QaO^` z!Y~{r$`(e1-_6l{C}@&~u(zzN7%mf6dBjKR_q87WcMZxsYG6bFavon5;1#Mv&m+qp zlFBBBU(hlW5a@oqka1kqu3Ry#ITABzpSCPF^zTK^<(SL!U5^xfg2p0;i-TCU<(W^u z8=9J(UZ2mxdz8IL3@tuL(N9EM^f0(MubK6d^MMswS(t}j9470 YL%c=WY#aE%Q|ZL%adL-sTvZ|Xe+uRt)Bpeg diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_4/.zarray b/tests/v2/fixture/test_format_compatibility/array_10/compressor_4/.zarray deleted file mode 100644 index e7a01eff8e..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_10/compressor_4/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 2000 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "248_F6jml-unyU zIOiPy`OTd7nh|B!-Tmyf_w&p%4;*Y5irZrt81UyFCkzY@3=9kg^fzre(=RPIzPm-R z3@Y$2?(Tx_+Jac_JK6(+C0HHq4|j632?zWIPfIwkHaarg)gR*S^yGMJ^AR2le`Z%)<5D(d^;P!S~+N?&HfP^gVbrz@Ja-f221gSVR(d zYB(xf654joVUj+9CGPrZgG%*%2Mm{4#w50+d*v3jdPEo8LyN3wTq)1$9a_z(9t2*i zoEdx>@7g^&?U+6!KAYS*LRsJX1BO>|11S7<`3uDI{*6Zu^b&?p1s#i)NfZN{Pgx9; zM=^xolzTM#hs#d+Y+q zMfbav3R?lxpVdzyn7a(%k{*s&pl1n9JK+{ft$31Fzt{<)ZPEN0#oA+pkit7=g_$EV zzvadX9Pk&mEv|FO|!;nBulp;d@|3d zB}TUJ_cwSYS^I*a)AWvQxH?7WQnAY%p8X^y)yRtJJC6e=Xl_QNjm#4*xChWk}->H z?tOP(S0X8-Zt(?-Q`>yLsh)cx8ZpQGwu{i93{E!b$3x!{3X6Jl<*Wmt&}jyzHr!Xm z=TdPi9G(NDCe zT5fCQ|CEScV0G&yHmX3BPuYCrKX%W$5mW8skx2OL1LsZxy;9g`32WRwL*(Xls7jgp zf+17%j;%O4MZccLF0*^|lNeVaE2eEf4*W`E+k8tS?^G;up2ek`$gmtHT=Y)ljZ!FZw}8vNki^8l4+OFV8cic4MC`2Xjn%-tV_3 z`C@`yt%Y2mc$O}?6a9EGkZr=9w%7Jf^J}b1O{Bc%r>WvwQGE07SjJtcdu)iB3~zo? zN63{sP8QvW;GOeh8huOEWkuMak5Q%ymm_-3;Wrc=7WnnLXeG)pSuzJ# z7~Tw57`3miF!tiEFz5!ZFtkywFk<+xF#7E7USZ@bTwyE(TwxG@zQT|{xx(;gzQSlU zxWYI}y25xca)qIXeua@B1l;gu7;*w`)V@M325#)d39JA&=ms1}fg9Q=g)e{`G5kwG z?2sD}MvF0UBVXZX8gOGFfb}bIgZQ)2E#QXyNs1V7!=HJ~1+!da-I=mfWi5=nUFjl( zv(@z1ho|2>ZdXZcyU|o=Y)0PeR)Y`c>9mB+=KgMumnl8vg;S%m8+*TB3px66uPtKU zqj_thk8-nq1P#v)eHiO>p<yyHWIH*`B08aWU{t^YhbpEKP>T$xNgA7>UBeZy=K| zhO(Tb$@kmi%?0sRYeQdgp-vZjDbZgHW;#lf?X|_33E-~Ogy?YIm@4vmNq0Vw;UGo2 z+Zt>71ZTM>SepZ7ve5Iz{a^hb?4OeCeu**R$6l%q(qc#cR^Xv{|Ew?FPV(+fOSJLh z+ly6!T5L!Y`R)p|KYP<`CGKoDN5AI7TBr=rWJUU#=k}cDv^Uj8oOtVVlo8*p`HFWM zEQsT|uJY6;Jt;O~M4L^KhL3K}eezdlMi|R=k-K-?oop>ixX~D4z>7Io?x)5CKl<^l zEY(q0l9dSIdPBHA55{bnuj&K1ksN1P%EQh?OJRZ^_3!l_qR*82sxZP1XTNzyanO-q zA%wqHAEwKV_PxYMnGptZLx%jIE>suz;H|{4-xd!Xs112Ve?ACy-)oK2;kW@4AHNDc zn(gssg1D=-?JV(#X5)f+xUQ@6wc+h&Ec)^Nh&*QL;~0YWc{8|@u4Stv^1ihW;rZ=z zXZS2q`tI@N{cDJ%JPKz1>Gk7#7&L-v_F;LgGe2>eCG}im%X(MgNx0=rebZ`<9lXO+ z@~hiN=C;mI@u^t&rIhup-Gk$^%bWYB){YTxlQQs#DQKHH`$wf0*LRIC?!jOX(QpVp zQ+sXi6_%7&)jBk@`4a_~l9^BPrJj{*U~E=dQ}5*JA^a^8dTtSUEmJ4o$h5-R1>;?Z z3vcvr!W5c(_G*E4nGw}Fqb<8LPt;KSM5G_@=#LcHj)>e>3WCI$beWy>OTOxaOkCo+FG8Mg&nR zO#w%ZVEeOCwfWcE_P@N)!U&S?<*`=rw@MGG%ouFi{B%bN!A+pdVgAI|B02DKTJML| zu`5y_b}U&IV;OIgc(3B5&Z_z0TljakqDel`7jrj=bjyF$S~A^tg7w9WBu=9((YcM9?*ANF?NzvCv%m-qV$mGZ6mzcyB;is=x-$2 z87uT=B?sKj4RE6V-8Y{{eZ8xdzD#FYywBxS4=e1~g()97POO=>c;BROkJGUZM#!fV zLpEZJNF#ND#<*aI!=VORsvZ5=`%yx*lzD1sDK`7+Ujklj&{D$$s!4PA4D+^Fp{4je zTW?5*4na%JP#JvOWU;CI`Plsn%F~cRT;+uC&{FHn)}K!K+@Pg`25?@+Pm}9q|A3Y{ ze&qTYNg@zhDt3zORn{7#W!Vw56oPm_?`_2xXsL`<2J_OxhtN`@?|N=2L{E@te}I-M zKHz@S04D-1B_H+mj#l~-ooNxYR2{6aZx^NW`p7ww>%mL|);OP(PU88d-6)#%kh3M%gP8iQaLQkv#66mJC2!V6 z%9i*M^q}7gyZps*9PgAX*=JqEEQz(i2Yr^JdEM__Lj6!2R=h=6ZUo1e zBT=meIExQWhcSAA>`pkxr~^@r`b7%3RQqf6{AWAw*+=aOYt+tD1m?Y|!KGxj-?NR_ z6I84HN*0*&qHZ%n%LA7hwj-!g{gw1&&huWIA!?rVRv0U|l*(BW|EwoftKp4Y>CI5q zAzS=PaH$y&sxJmNK1ywdungJYRVx2X1ZVl8kMi;9MhNqu4Q_?d9xl;w&V|RDeUr>$8nFQT**J{B4tVZ z2z=0Q2`&}OJLN+5`4wW8_*x)ipXKec7f0Yyq)o37GR0Q|7<(;lmnt5`@JxcUAY_QG zzGLV$$0}7gjCuI&EeW`k=*m0#9`joz3J1{-C!9$dwBbL1OLd#wDt>+t#Xa%nZi5!w z2a#oeI&dlZ{V1-lZ|>Ac2u%%ywgo_Fgc^jZK}aSUgxI`5h`xdg$hDRTOi~KLimOtbS4c#v#cQ03PLKmAjBUELR7XObOVGE zH$li_2!sqOK}ZUOW>`Sz3myo4{0TxKAVg^cLMX~0wDA}UxetPnegz0UO#mTg5Ng2% zp`23?3jPj4Fg`vV(A&{(H#w$#n8sXb`yI(UtcYeg{HIxcCYU_O=v{|to%^XAd&q4& zNUU0kDrV6*H2uq_QCQV{iY5@4ls)suVCfZIbB51p<()J8PpM=a(|eA{pV}sO?30LF zCA4l6QB+9H+M%UzbZYMVla@(LTcaehwyEv-5EqF|m?Ok8wJ2|T666bxnZQQTH!7~V z;^goT8=Z&K*2ynBV`cL8>z#&BRmm(kVy1KV=o|%-e|kD&i<->dp|S5zQYt=Wg`B|B zs=Dn>^paw|!d(h~HqTA~Z8XK01Ew=llWxDkUx9SB%tZowI@_8LWiZiz^{g#Sjb^9T zM~--@=#9v&i41cdguYl^rjwRnWy;M;Pica=e0xFku~ZW-*zPE8`oqQmMY6SWS4o`j zIX3(^hLeog&fCM)X?N><xtH3IBE)fLB9UU{VCo|uI&@lk>uCx7ab8A_xJ1lpOdVVz7@Ycm1Xq^c_2Zb<*YSK zm3q6zTb9V`B_9Ry`d~$ZyMYu7{@HA6UYMP#05#h7XibW@u`CBM%wlIzq?4uy-Oc@p zh7bPc3J*yUR(s20U0z8%z&@UC&JMCx;-f^_7_3b6Fpy@&KbvdI3$s@fph4dmtxfeY zmg6LbS?Vf^e4`~of9qhPG2@-NA`dCTT3=bLtF8nS&dGF3PO!D|V@j0Gp~^&0LuodG zUvusG@9otEY0!7aYEykpzjgSnF(bf2k(Uf{t-n0ZRZo%$=k$9^ zPKb>%Kh=%R;i@E0BN;Y=^ZEAta0hikT8!QCx-?%?d9FKf%iYD%&f203Scj8MnE{qB zc*&4{^q0rG=}9upx}2`#JL^H`G?;$vxEVk?Lgc*U#9AE*3fp zBOEn^?_=(Ntxxwid(KS)ztU40^HxWk@%Hi5=d2*BmyaO(2O#_PrC9JF`*R`tpFsAH zK=!|uUqEh=^L6@eBTnu8)0bcr`T_6;Va8dyOK^I3(q zhk2x72zIM#y#i&R#m=iyiJAwg*gnU`)3VK+poR6KQWswlWh9EzhSs1+C1`PAFd^N* znt}iB%zRFvjR&-_UsdZ9D5Hr*ciGV$m8bAJnmKcbgJGARhD0jq z6K{DO7*0qxf)>Bb=M>v`@=4z#*srPe34#``JGx^MwM?lvJ|`yAaxI_*to4#|*W+T! zD3mvwT0^3sh3A3cH|a*U41#yREanv3K?_9tHT6Eha+)}F*InH)NzlUg#N@kN3s=sa zU|5?aZ$30f48g(Erq(C8N`r;W#O z+XF3_({TMxO=sl4aDBWR0%r?alu$*ZI&bL=i&ZfsV|g8c7VMdXf#;SBFFJS&$itBw zL5okc@tAIV`r}XQnA36neu5U3%$>J7`hQoQ~)B(`@GX7w+7? z5U*xTx$S-Ydg|F{jx3@e(4zBEAw>ju`Uiw7?t`aa|Ep9z%LhF0^gnNXmnYjK9NAEdDl7WMq?cHoo_vReMik#7Wsn=4*w!;O2?hg%&nqnG z<`Z7rYhvGQmwE?nCmU)>oE5r?a>d9a+Kg5B-Se_U)#H5Dy;quH5bD_fg*I7k$x z%8FqzNeQddi|??=h+-0=XmVoP$kZep3=lc5u%w$$geW|C_X`ZjCc>qV8XQ=Ee9(X> zgwD#%X=mdVFxuQ6?&ax`LKFyfcFe13YPh|A0;grB)H4vp7Zv!H=PKn`Che52-#FiINMyjMRbZQS zQoB!aM{GaMg43wNwd|(#{LUM-E1iR&Rf})kOXoey7qkB?7g?vC(6*n!Z=OHyP(J^L zZX>b%(1Ul20*S{8g=qSpNgPL*16PESPhS*c80m!8{U8*xP3u2rQ1LCF$Kp4MYd@rv zbIt#T!m0bdaf4XGK4TE(ftr8CBDR28Lgz8{bN9k2G;aNf&)avOI%W;SF>3@?E#nGV zCUu|EzVIymj=^gf)v`+}{U&D&fmJKGb`@X5I<@zVPRYA;_9ox!nAUxYXK!=ABD3p+ z)~`dO)4^yzFxnIvO(hSFzQF~J-UOqEpwaItq0#1v&}aoP`XLxi0*xMlhelVgK%^GJqfm2F>=a;bSItZe=f6ZklmcFn^e%4)e4 zSG;`TZ&oPDoIScf-`{V1vEfBrA~9o+nZY|~1QY$>i`t$)S%u7^GxkS;aWljOwodKi z5bC<;YwiSvqEj|#soZ_~zar>AD{uSUEqglWcq{AiurXXLbDPFNAZ3;8vI}mW(1ay& z5=Zx|(=ghG7weux#p2)XFh1}O7+yp%wy5sw3lY ztF6(~d46xb<45{?>rvC+TT|7@|K9pW3fI-vFE_n^Z~bWK_0`tz$SZ$uZH|=qduxRs zzqf`={k=70>ebeeslT^|OugC~GWGY?kg3179yGe>iqc^`X$e)O*{$+`2+E|*|L&W|B1Ge*yNG*n==3te3+qJ&WN!H4t&z$oGP9j*7+1C81BdI3b z@O^R6=eYK8O}fLTASKGJY9G0~D`l>dc(eHq!Z*KVTJfO_CmVCZ^*|pwf406?zrWuY z@PcBa%1f4bxx_^RXC~KP5c6w>1uybok`V_?ceKuflP{raG<)^_ie&2*o-#y>#m-{b z(;saG(8trwcn}8?4B0L^BefZhn?qEnck29}ldgSomnK*!bP~my%C_N08%;IkM(B&z zXZ_V4p-F%EIY^mmyT(WU?rNFaQ~bGnN8wxFvaBB87)~+fg6oadV?JwpuW|pNG4Lhj zW|g<>o#j#&N!*z{dm+q;49iEzLrJeWV7p^pF`a%1Q={E$cn5vBt@M0Gv{d|79Q*r6 zI{}RG59Yi`1Br(0m!=-Kgs4&jQ%Qe(@{k6mir${evAHr8p+DY$6_`qY^f_1=m?}@Q zTJCmb>aEEv>&L)UF8JO!Jz%N^-9b|zFx6Y`&PwT(sX{j=GA)6ruQ_3RVt}b(>a_a} zSEfEAUMhJDOtlll`1%2unrOrh(*;buy!Dl-(5)AXfT_@}(Z_(Pm$$w$6}t6KEie_j zHNiYE_43xh)XQ66nF`&SY73YO-TEmWF!l1*SEfR@X8s9Gg>LJ4ImX<*lzwg>G%m15AZ(9i`0(nRx(S)!LA*`TXrFsZUv6{?hU9RFTgjGi9x(H!8Ls-}j)4~we zwvlp|+N})lkJX#+Ls*UX0(KxQKM0G|YfToyT9zsy zaG94Vgs?<&vFsr%Ob9Cj)nXLFG9Bbjg0MIstZtU*Ul3NriRKsj&_gu{Yfq`33c{*^ zu;gJF8C#&Yqu*|}J2B3Li=O0ZV71uOnYEKU4Kw=A@Px06CZEXXA!<}&8Mlzw_tQ0* z5x3ir_d3xGyE9JsvdskXEQLQ=j}_fZmO9ChzsOTUEYU!#)WfPb#&5AC>9nWne@kcF zL}1oJY}rm?(?xFIOXV~`<1$R=KE~)Z!R$N5_HKqNXr3o@i7$NhNz}Sf?6zpao<#DY zRNBe2j5GP13&lJ*r9wp25)_ScG@VLJy&5dTdK}{>e9*Fu#HN$nzK6=GpXTi_U86Bx zlPN*78DWbBajWH>c59Lj8`3U2@@{*|UPr1vC+Y!bnn9QQ!>)8A?hIodjN@KR6W+{| zzN}MzY~SB;%mi@F1#!;@^DKt)E`{-}gg;)5eDWhoU_C}?Ggf#zUUVlxY%fV-KUwlH zRq7~B=H$b((+s(@EcstK3Kt(0Ve($We14?s1*=7YOI{_s{51_o)4^h6Qg(%k=BaH>{)|;YD1qn8qW6gvK zw_4&XM2NRr6D-B!;Lv#6|Bp1bFn&9pBHPXHq3zU_DWs2(PNy|hRD}XaDOyL znF!#oe~vK~BG_z+H5Vq@{t|B?O1#sSU?p~UuRY0Hf^@$#*;bP5uq)M0isGmz%|V9p zr1yigO_3gEG|&Ad1KfDNr_uxXuZ3PJOo-o# zd{kMGCrfLKPrRfrzXt;L!!{YO~Kb?*)N zuvh9MjQDYYmrsC~CW81I%`s*|z)N%BWxS>6o!z!XD{+#&jwIk^C-CyHJJnvA;=t=_p?9KS&rsxF#GLuTHvJu-No8SV{W9I@e2WZJlOTIro3nm)0cvb1n?W<%>}WZ zX3D}$ga}&`EroI37Ahjl#7NuUT8k5WEY-wV?;!6^wU;FNS!;;1k=)py?kG$4x6zd3 zAVocx=_*eNu+x6ZNsfLr+g*_sWUnL5O^IU)?5HdAkm}~?LSJ=8sFR*7FZHdT zO9Qo;Va^8fk7#cHS{ba%4tFtp{`fxj`RZ^(PK4`gMSeP5m>&>mlsg2eAatKrlfF1(S zKmbYsf%a5Fps!>g(31rSG{gx4y$3*R0H_=QU4}s4zlA_)u_4g<90*i_9|Bzipl$#Z z4+3qwCnK+^%}Ljc+bK=mNdTR$Pt%rFS_5e)=7SO0E5kt3HZ63y+I5vGGs|VQ0kK-vtUK?K-|iYr$+z>B&?aL0{jy%s7!t2%%OMHB z;j(Wut5Hh%$-jegW-8>Hm}qxRP`X8Fc{u(Q}?S(%fmvd*qdIb<_ju) zBGpV8M4^aI+j`$*zwj1O$KUe#X}P32AX>|kc_$3ndB}THT=op7pSM3Qy`O~be{Qp?F)Y!*obn2>yJ9DQIZSS@8}xSc+f5FO+(k0g z$bK|Yx0=Hkjv)(-T=7LuvO3k{NVa}6v~1CNH}b+8dRmMLkbDYn1*Fc4t1~Gp;K}Ri|B?W z3=TEV+`$uaar5Z4=kRbld+4bntMcjtQ%N7MkiwvQ#JF5%BdSzI7TOB6H_tsh|H<$UpPi1WfN5~ zbN@ensqJ4lLL+4rRWWss%&zU9KSCp66;(EMkI1g=n?FJ&VHHs}b&JTZ>6<@9C1DX! zHgSu{s_C0MM7_%*qGaM0o>kL3cZh&VK*7MtFY#R6z|zSpI3_KxyrE-gYI*wY1_l*mctt?Q&JARWYd~iG5M*drKt{&|WD>JLW_%80uLa`Pfb;Q^Pigilk-0{OzaOIU)Q`gw{g6#$@o}U zc3m^GxVCj&Lriv$?z-lMs+Qh$O;}WX%5_cm!05N@8f0`Vyz3f4F)6t}H6Ff!e`@MJ zxBaO({ss4^hK-x=PmP(i!`thcoPyHI>zakt&AsayQmXq8u4_~@b&alTBIA-jT-WrE zOnkqtLA!-ZbX_C*R95l2#@#0%^iNGwYuBHeGgzcQHC&Gb{?u66JO8Q4FR7@#u36dG zJ-V);q-A8iuF-g9XnI`}o0Oh?T{HZ3dj7iRCJrIVb&cdR1?B4+?{^^)*ENk_I{W_A zz#yXjso{Mh@~6h`joY7^;!ibyYSwoSe_q#IKG?r&E+6dQHJ1ax|&1(fSHYhn+MDhm3S&6`&{vAj=qtxnT542 znB(r{;~x+ldNn8YLuO8HLDAKmrk2)@uB$nd-)9$=R)1X0IlF*GK)P``2cPH;DLEDO z{mVIAJdgMVuI9)qzEn}u)YiS6V`Xda>V86IKOpqb#r(3@bvWZ^7irZ_4V`je-{uC7#I{B91;>58X6WB_Wu3*a7Ymm5s{IR zQBhIRkYZwDVq;_D;^N{VB_t#yCMG5&B_%^jxm4~oD*jgG->Ujs)qktzZ`J;-y1!Nb zw;KLd<3H8(PksKUn*XVmf9lIW)%s7h{ZsA#RL4Kn`A>EIQ{Dem&p*}sPxbv<{r^D& z|3QQQK|}vR!~a1e|3RbwL1X_x{Rd6|2Yvr9n)zSS?EjMH z{+BfW{}K%U|5rUGo3Z5)?AzGBsJy9*gzQa(5E&tQ31yE&h>Q>+gviJYW$(QrMA@6nZ13;> zqtiK`a~{6Wbw98$-{ASnU|_(%Unr zJk(gWTap~*C>N?DErswV3Ih$ePx>=GRGGJ$6CGqJ=Bpws1aZd;1N6C$`!d{BnEy5< zJdz=w{~B&CfHPL$uYdKZ_rp_Vrp?B;_R?f?mErgKFaG@Or^k8NlkRqhaicNbPKtE4 z;*}ZSh0%OpU5>-XA;y$PL;ec;yEA6ey(wOw=MaxJoWOoC@T?y$uGfoub&Nme5QVRr}drF zEvluO*AIm8e-#Jag*80YE-hBSh8>Cm4OzBYV1b3INDCnYu9M~j2YBxYbHU@@40z+F zw~quaj={Sg^?b0GB7?W&zwk5N^$xtBoh0dKo-e#v+J8zU)8+6^BcFVCHHARI-;X2C7h1yq%YH56^>>uh%;Ugpu+SQRZ5f3ef8J7iYncd88<6m-A9%A ze!83+4e_XQrXtLYocb0{y)O^n(+q8b~t@NH7?P&K~F#=*3LFBm{R*TG^oJj)g$5|^7$^#e z-3AuA^v2SMdz3vPvWF!O98(E81Wi67o1Y#q4!^#G zQSHjJkY+^D6)f|w_!0ZBcrCmJFQN4u3;Ll5#q+OEuFt(UBU`-1=vNsDf_bmBML-3-?Tl0ZDs+#4f5Bi&48+>%qU0 zX-3l*CVyOxD&FZ4wfc!|=UcOkMyq0fb4L{>R6Q@{4oe+5ekbY>Hv5R83Z@^gl`*T` zco))*DY{?C9H5HHw_5m(UcwtWmJCCYO6OImBK0m=XOPr>ksaGaoF;Dlb5vnYHxRCH zTIs?)n_@uP9w3P-E|0~iU##`yU&%6~=?{}XK^4>Q^oU#i#JBSwvW&h_#r}>eGEAs? zU&HQ*mwu$&`bGpHBg|kYRYqKeawwK3Z)i2h0@~>u@(Q-|{(yuP62Q6uqG5F1^eWCH5xl=&-Y)T=I^y$Q{ zY@+{uN@wCf9+pVI=rACff@Rkum~?L4$rpcY*~SxdaKDK=YWH3}XZY6L?`)wPdR3QR ztY}v-1}tin(D}}(6j49_qx6~FV^Ths#BEIWBcaQP)CXLrA+gj8kNSj@F>Ja85>Bn! zZ^Rv$w_Ja-Z`yb@a>uy#%B#NyHLM|Px?h=tmQj{J%A)n2QASz674pd3CvN4SESXGv zXOxA5vfg9a^`fkYoqTUkEZcZdRx@|h9?FV9S?r-3`Y3BvyMhU2m7*-wB5E&`Md2}d z`xA-VILdl5B9(!%#8Fnia55&!N<6jdxEXh3fwKOYHgch?I+SHt!}=0su>>t^l{5G+ zpe&ymkMVZA6c}bEVD+g z*Sp4b94PA>>&tbN6^ybN{TI|x)~s?N)$`wq`6x?1hu9Tm5umIOIF5rT%f46W9mc~h zew5YD8+(YdqW4T1u10Jdp{z~)>dPpr5@l(Y()*z-8n5X)1r(lBDC_CCTsF#*Mp?t+ zX%|se3Z`w3VB(oI$~v}axu2d%3Px#S*-NvH}*=OHh^y%KD?2PmZ#3 ziCxEJvj|*J7Ovx<7|QBHSvFn#2`KAE?4dcz+DBQD+eWo0%b@yl$eKE)Lc8qGT zpsX5{rF$)%`W430Rf`{L^&TsGKpTPSvG!n8y-~D)ErB@jz-h*) zNB%2sCS4@9=Z4i!%_i|e_9T+v!^d;ReF{}KvlyZ-Jl}jcuGJz@%<+!w#j(r0NxxDJ zUpC_#9Iq{#UplQ)U#_N*zdZSG(RA?6cYz${7+jxiyGh-4=~Axul%Z#?OZSIVY6U;B z#Nqqy+W*$?kS)8GM)m6a>5BOe)q3H)%khN%dk%jLyW}daf1nA+bYHU^QEw3a%$h(H z@XvAPZnu0TZw757mgo9|pBhbK1?-6=K?jd#jeBob-N>SUec{=r^_W((MA4OZq%V$~ z=kN6^R`Y#ijK1jg*JeVyRq_jGGI_|!lLgZOrEmN>OmA?#w{3svwn>*J5vw!J-tfs2MDp%N&E}vtu`@*Dh1al}Z_Q=DK`;Sfy6z z)1^3kzuiZ_^*d$Ducg7cS1e%Bd{}fJ7QF?FM!=$@uqaysQQ$wPnY(cA3^?~XoLdae z9R%m@g>%!xxvj@xQ4Uzt1s1J=MR8zJ9auC47F{$QxbsaQhY8MY3+FC@bDzSwmEqj6 zaPBEMH+LE=x&n&|!=nDMXcsI>1B+UYs5OXuW_=6io`!Stz_~r(+>LN&i(t_sSX2=fjfO=hU{TIwvXJ8^3np;xY&iE8oLdUc{SwYS2z)iktq4es7Ey>Dq{W9Q)H^50XB=ia^nK_Q{xkws-aP4-Sux{hFGdU07OO|GT~W@9^Xd6B`GQkeHN$ zik6;pJgE0U=RwDH-@Zsv26l`ge^VIXVC5=J5=E!;8>Ykx?;mA9M2xib~4h zXV*8iw0HLO{h6I#T3O%3z`lq_z{ql$orCL|q>S8cMMGng`xaIY-Jf~+`oD>Lo0OcI z{-vy z@e@~fPcI*T^juiP>o>7)6F+?}{8C!+wFW)i+}6?EJ1{i2xV*NpwQ~U%pNQlV+m)-_ zJU3+J74E3qGc$i+ZEOGB$1gBAKJnfAw2Z9suhrk{`iFk}9H0FCcW3Y5==2;5Jx54F zMoCS_aQz0Kps<*PG-Y5B!t(0LIpzf%S_Y;|tXDYAE#IY7f2(U~{xQ1u@95;5jDk;Kc5Z%QacOyFb!~lP z^Dl<1|Ib|hDt;_ZKAjL_>nOM4po}bR3V&ToT=uQ=b5;KUmPC@~;X#hT_BW#39RZ>3 z$y{A>3UeB9SrMK*R71NeLt6?hl@9&Q#aJA{dt3&UoHWz>%<&-&OU%A8otAIAmqN4c z^cg}1SmWEc#ffk>XY+9=Wp4L-W=N{1lH8gNQ>DsY))S+e4Ldq29NH_t8Xaxop0r`SaVbCXa~Ulc1=}loRcp$|7Xs_`gqSJ;F6G)~mV-L;Iaw#8b>^yKvgIR69kc`6tH diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_5/.zarray b/tests/v2/fixture/test_format_compatibility/array_10/compressor_5/.zarray deleted file mode 100644 index ea781710a4..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_10/compressor_5/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 2000 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 2 - }, - "dtype": "e@ocWQ7xAFD)v%4jsZ0FHWt>9GJqmo1cR6 zI90->%{`Z0w7Hu-YKqGqHKjWjZ3^+))8f^jd&NQ(I`L7!qC@3Y6bsPop*np)!X>*s z+MD|h-kD_AH+Q_H@@h~^UGdw~fv&cAeYbc^rm-Q#UWz@L#)f2KMRyW`V-ldY!CHN< zfHnN~lsv6=z2S8N=RBIVoF_imsk2#V)WcPwQ6o4U&dVa8;!YdKARukK{C zx{vL{Qf(icRjalZKiNZ9FPpOa;$Cy85J&*LuLL#;X@6Kgr`%erHWrHxb+w`F>pK=B@$dV-FUz(=AW4!GGZK3;jjV)2 ziit#sJ-viO`rz@Z^s*_VuAN5Rokl}gmKKpnBpQuIV=NYn#Z1gI&-1(rYumPMr$CY! zQh^ZX%fBFVk79zr(4B1d!@{zucw67A2po;9r^5Tb?>U>;HhUuf|NsBt{~!Mq@~RQ9 z8qGpxE-j%G-{}D=hpJg92#EUTRiufEC2|LP0|s3)klgCZ-P&@vhqY#*Zgph@ee=?L zG*L14;k+;7w4~wOhvT%2Q~7)+Xip$P6BT9p=H>ItI4xy3_u*Yy#wqHSSEZR3rZ9vjA8svFAEyV91}ObIkb3k6|W; z{8KbX4nIl9nDLV%!wd}hr*w`Se)7YZIT|^g;Sq!;8gS^_Kt~dN{wZK~KQQIJXuhg|x zgIaA6uRZEo!|ztRqEf+JHhy($wFPUR$YqmCaXH-LC7mAOL9q6))*f1$hqiXBwVQ>y zSu5^owNL`ryc-L!2}WWJd#*#fQnOa-){5uUVBK16BhX$xk;_I!8^PcX-fZH0;r0X) zo8s*C6P>(hBYmT|fzd*`s6_~DV<9k=!W}%7?_JQbfFT3+LxK@x1u&Ju9lTpCEUiGe z+%}`g^!b_~sLSI8arG!6YH37Ikb)+KO9)5ts?eybLZhx8<>hA&cb83&9_3Y`QP(b; zAhU=2v)QuZaDQLqLdb+>#r1@Qeafvjd(w@%OoKxbi5Q7RU8doX_!-5d>y=eWBpO!S zqYE%X!y$=EG&ZD|2(j3cX*e5E?1k9VOEfs7n8-`9CoAD>KJ@PSg7mTq2zr{XeePv6 zq}8EXz2OZ!zd=afRFK;c<9tDS+T-PC8Fl#?#dN96=j+;Oj~AquJzjoBL0y#@#dQ7d z_CD!Y?OLz8lg+a5)lT76qS~sYR~_;Q*0!nlnqzJDgR3=4$9`;W?qsvD>QgU;kqAO| znsQ*tr6NQp)8lY_FxUpiPc`C0d@vK(hKl2-JKgGvy5jn-3L(gfB+QQihZG88I2cJJ zG8r)*A^!i*z>wrm>N*@VDH)yrKawN|y5lPoFi1#}B+1PGlO*}^|3;F`V5#o}&IM5X zF(gTj{Qr<7GyjnMkK{)TsrgAVda((-&$U;Oe+q}+NRGgeouAAYJ0vif;RlDKCI!W= zRI_nZrg$@cTmYjPj$|A=DKQz^Mxg16Pm!h=^rRLmd^phMZ4d!Fu0u}gO8 zvNw0sw1Z_A?fPbq?s$oZtLweG)p&Ic^0Z2A3k-GuJLlQ!vg(L&TGnEB1rX#}mDF|( z-gtO@C(sgda)g{C6W@D|&d|(<=GpIsojO)Mc5cGd+&vv$)j6w9juXD+McKBKjobC- z&ao_;P>9*c3Hbag2a*H$MfXRynYA z6Q(fEw(DtAgY~HLD%CkOCL$sQNs=@MVhbP)qcE(NGanLQ@SsFVau`zxA!7^?5t)%d zBoGNADM&$*#>xp=j7RvJKCP0S;qZg;^I|h52M3T8c5z>4)X~=~&`94_UO2&C0qm~~ zy9u;psuSCbco)R@J{3`2R9OY=NhI}`7hId}#afWH^Ft`tO?(AB{IvRyDInxgR#s0| zAaKp2dQgepppe5KmfL_N?4G5ElkPD0cVju`G1;Ck{F^L6k%-Msz!Uo!<0ek&O6?9t z8)?z;df)~Y^ha?z*EJs@Vd3#E88!AX`VP~p1F%j7qc=jsTNQu+MGGTTQWT2c=TISl z@Xb>Yy_=$&ik8q+7Q5dR0e~Vk(;on3{%dEyf+sn8TGA@3iWVfDbm5tNA)J0>)xYED$jT`|O4Ma8?d$gdUph<|Vvf)o5`hzLu9b4dk=>n=5 z5NF{7Z7n~>8a2l#*e-220o+2EhI2yXiT?n!r z_3Y~{G2?Ps3QtAK(B2%hI)cUmKROvnGf+lC=*s48=|s@WY{LZ4sr-Wt%0qmH zt+1PE%~oc#8VxR^!#SFB(lo*Us>WuWZ)uyC-545gvw(Uu#b>d)%zhL)MMu@3Gfhtd z%3k_T;eJmz0}V*HE|YMi&<=8-#<-lp+^X5mi2-K~djDi=T>;XksN1Arww zQE7dEp@|=CpXqf;YX}U?dx&k6i&aSAz3|56_qUgo0r|3B7{rcZ*k8fOxkGJg(g`Af zz${cP8E7}N{Cd!DUb=BZ7wWsevE~zbW>(NxdcKYAkUIT^zh=9>hUbcPrH`Qv0rPSoS?!U!zIHI9qxYb=o;Cg>h&Nr(_{-X)Zf zWM(9o-D^@@7M0Ip7TVb^;;J#1$P+2$N%XKTc>Z|Hf3sUbfLnF;OrAUyC~ILLf@-3^RTjqx3KLpDktzWJ9vx4h@-wS;2uw0y*2MUj6h^I9WwX55=;9rg8g}SH?6kuR+x_BQyq2*rT6!i6V#|pPr$?z7xPkNZNtrrdQm#x9u{@OC1?@9ft`f}`ujo8i zvw*xGz&iVgWr~fkZU0-f2-TCQf~F$t@QY!GLUFy)*J;9at+oiS zB!T@$h++vH^hd;T@gt6@7@qI-oX?z!*S}^AWmJO@@Pu_u#W(bk8@CP)d+Yh>x5ikj ztFb_n46hGVTFzy^mlbPxBdNQfK{(DVIaHtV`REVEB=@GQpzEsGv4A=aGVMg~28ktvgPB4W_Y`vaYSOfERp-wn#*%x3 z6GyrRizi-xOj^rym50(%?#yQ!I2iX5H^t~m*}-sQSc#C#%(lh3cZcR7gx#D_2D-}l dhU?J<{T0001_2><{nwJ-f(fIZbr0E%V&6+Ta5Vm4XjI2bX^%}4`rnAVZw6G&) zM4P^!^WBtn3RnS60Zsu8LUNnR#KE0I(3u?E3J_p46d%$mhFOWwVk4T9>pwwx)L^l(F-_@PX(P7j!5of$A`b3y+J0(_ z-o&vL0Jeb0lXKQemF?xiAo8@q7S37Au!zo56^qEy_Hr*E1GJCkBob;+U{)~_1D%Eh zSR50t`QZ>s#iVRP} zrb1>?@G>^cq~JZk!AuI2P3gwLOdW*_NF`W_n+sHL>b#2RjLuq!bjF4$&4y{F!$-(H zIm|aj!V|9BSu$c+Snl1V{dijQJ%%G{#tZZefOQ#D39U{ z-oW~+(N_k)C8bK-qt(_@FP}(lrRV6;vhxb^qG=R*xdss@pD)YnSIP-#*`wRX>V#Z;f9~_44w@rQy50 zKnq#MJc)0QDRZ82;i%M2G5owk>2Axm2f zi_S8`wQx>b41>rr!@ZnDXa+i5jbKg`8Ziw^sEk=>%pyHlq|FuC+=ns+lOC7!#1Ed> za%Nag4>(lDEYgUUk$LuML|d>|Bb3JWNGObK#QUH|v5d^Kc>!1UARN$TLsd&K_J)`2 zLC{n}ozb_sWnC*o2+mp&omWtTg-B-U7$K+!&IGHh*aXXLZme7644p! z0@axfQ;E5SNa@=urZnpU(HR?p>fFK;p4d~e%GZ29$oOP=I0C%#dFAuU=atVZpI1Jw zd|vsy^7#SY{T%Q9%_l#{yMOcU-+c0OeDZnsZ{GczcmL+yzxm|n_~hsK)>bf$(o`LBcDXca%)-2nHb$2)V9fLBd_R zAer0|3`n{Vi-cT-Z$`FPk-Kk|y zXSvhrtJyFEyKm^NLBhRDnbps+a3tEFjK#)@R1t=*5~W#i>*4RYOO5V`JhJ$_!UL3Nkw{`v9q3A$YO4L^RKU)&K7z`u|;2|G$gq|927n|1PTk-$nHQJ9S!*2RF2PU+!c5>EO&}GUJS8 zmJ0loiyl%IWsNQ)QQ!-Qdb*1=a2(7SGwU?ak^^{utaJwl(7?WbciG%~Nm!Y5#1v5glQ4fV2rR?=NeSt1A;Xf!=kSxdYFJ-%5r2szybfb~cxtAlcj}7WIgaf| zWRoFZQ?|@S{G5JFzjLlpDYP@1U>7!#AL)Y*YS6`dEgye&omyfy#) z*fi4Stx5M&rZuR;sEG&FU>C4Yev;yY3~Btq<% zOpmC<-BV)B%Hxa6e}v|P^nMrhvq7EP_>PSS0(6T#lVxi6qL#?f)I2k5p+w|QrJEEb zOU#Ap>vt=GqK?r9s{1@Q1+@+3vz9G~{{0WW;9zZyEqTJa*xN z=4RStUa6xLLsUNYnNE00xC*+KQR04g2K;S|2T@c!QZ}(Ui0O-?L-CIVulsH?G;^w7 zU^XK${dK!HY)F+ryTEQf`MChzi0K^yWH4pp|Ls2_)W8DS9nql!2hf}AMCcHxQAZ%` z>#L*`DZ4~H)`rD*393S8UisEIbKqa~#vg0L;=8DFW2MfM7V>Ta%I#{dTsG}rvW**5 zVz?6J6Kp1J>?@gU-cW4e^RmfswG9^pCt1IfjJ^PBu-E}aTTf|E&hziB4NfeJgCtA; zmyu#RzHPI<#BqcR+L|`JB*BAUb+5JrQKRrrJG75iU|xg85*uPP4>RV>zD~Mr?4=Nk z`K@y-Hw8edq9U}#!976V`WZc+l9=(xzzf8aS67Z0ct=e=EGGuiN%5vEQ*+m9?i#L{ GJ1%?BX31y( diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_6/.zarray b/tests/v2/fixture/test_format_compatibility/array_10/compressor_6/.zarray deleted file mode 100644 index a9fe1db2eb..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_10/compressor_6/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 2000 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "lz4", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "fFL{!WPjHsv=F=HZ&3B-tC1O!DyF<`&|EIH>KHs>%iyE{8O z8z$$}-D{4(qN1XFRge3}*QYrgJ;3%eeU!8QA0!m z!0pC2Gw#jGo{62&cb~fZ@mV9X4&DDIdhhY7`z`*1Z{yQ<&kJ5bp`@x34N$A5?( zH-Dh+kN9!sqj~e?hkmY{&qw+gADtF#S%2hXe#}pB>#RAuM_v2rj;!oY@w;)sCmmn; zEGO&Gf(6(+y!qLZ9SiulpX0fLH*TNW`Z*u_3!Ed@mwfy$5&qcuw|D$k{1{)M{>|c_ z?Ds9?$61KYdrx_+?{^FNu@>Q7Yocq$hl}{}7UA(;1vj*ux0oMuF$O=-dF#d1i}`W0 z(QV!n>dwZIc5Sn>@mpcey!c<){P;`I_~Qu&u3f%_%diC7`#(D2i|dwhIhNvsF?*Z7 z^)2PHEW^Z4+YK4|>M}0RG7PC2yL08K%ehR;(W`Bzx|eggmZRhRXDSKQmKZTL#;UbOA$WAj$>O|OKkME_M>&sBKlpNG`4tl|z>4GhwYv6@S~8hxMp zQ{Ah%)NAnP%KJa7TC(O%^-0$tpk7z6Yu0kz*CMCG;pboXtmXQz!{;~N8*;z2j_+X| zCcnJqjtfp+&-bw&cWye6`_A_Dd@t*>0s3>D_4)?Brww>%!_5C`t2gj{ zZ9u$iL*eVEZ@jld+w6_dKkA$(H}d6eMA2`D2Q=>9$d|Yg7o6NbG3)P*e3_fD_=5Y? zy@@Y%6S6Sefa&v5jUVw$WwsMnfMVVBJt=ufzaLJsPGC$n2jhkj0mdUk^n`awN z$hAFq$2Z%!iMC^!?8fcfOxt06)4gKN``fvxw&NjLhV9&3+aZ;u{|;`l9k@(Nzz%M< z9a!;jrMh=;)9t{CQ@(rn=qWpSc6VaNhh01Ed~7H8;Z9iV^mg9K{kRh&N}qe9{gIvA zm%C6E`09%Nckk+aLfi0NkhNI4i<@*8R(1Yb-MhG1ccb%w(?=CvvzwcCH)h^?<4I4? z+s)0p8(%;2wC!~7Zf@e;kZN|$p3WU}_Q3Xe(N&XQ-h)4vw>$XFl0CReRmkoYdss{M zV2%Ccwf2+ua*yxD$;s=+yz=N??)AOU6}az?z1;JA@m-fs9rb_g<=)?iv6tt+HRG;* zJOle6J7UQ`o`rq5`l(LeH(UF7CiY|f?5owipJ!t~PFwo;xQpN0&oi zejJt$m%}rY16@jc-jKtylLK|?d%yf-4$n{yR-W4HjhYX0c$RYDz4U~(YjSv`a?t7G z+pl;nki(;ugWImtMKgy-EEl@Ko_&5UOJpwg$VSQiq(gQtS|4A!s^PWVg&oe##kQ;i z6Z@~o<>|}C$eGpZ&gCi0#V1nja(NnaQN8d7^%nDZD)aEi+PTlZb$uRBXCAKH-Oc8C zB9Esu5325~W0uF$nupbOv)gye^LEJ2gI}(^Xomx*8<-@C5 z^6`_Lk6C{&`X+R5K2LN$_Q@jT^JM2kj>htQp74D1JAdSnXjMK>dOpSs{C4!fZ}VC3 z^ReI-{gMiJ`U~K=ugfXP>k8Ni3h>uB-5(2B5DGBp*_$1od{Dr`P=FU+_w>u`Do_PN z7uf==(@WtmVB;u2Xu)5>1%DN=W)$Mom0u3B&Mjo^D8wDx2B?Z#$Qn|J+4&{kefer3 zYe^xb6f7%bO(}%hf9fxx)&il8gk z(#MNfdy4SS8BLRadcTM@s0i2en^b%It|Hc=A}Ed0e}%7zHK_<21}}K~*CR!&O~r`Z zd9J#PS)+=f5BKVOidn0Qao0p$bBkHCiXo+MSutx@G4z33S6R#&R*WC#pS1Dt*Tt-5 z#W-)tx}(ESEn!V7!T62Ws=I`>tps1~bLIc}SP5%f37q9mp7GH8C9HKN_}6u4%9foa zta&AnBjPP#?JI#kAtR5Jum+Ywx?BHJ*1}T6{=Hk>rL2ji(7R*IE2XTBrI1p;w3Icn z6mu^;@#n0{Qr5~+=+$`S>r&RtQv58(^bWvroPcucM`_WCj`dO=H* zGM3dcD2=0va~aEP8Kh!(%UEX1aDyyJ8Ov=sqs7SWRBEWhPQ6uqqOa+cw8 zNNZbK&T?E1)m|4rQ&G;cT#omo*_E?AmqUxPE~iwmOjqF7Bl^HpPnAfDwc8(WYKX(^yKwjvu;@f!0pY-~w3^iWD;`Dhzk zlMOk;UQoA}ObCV6)v-I9$!#Pjt9c^O|v!P5H zk&V5~26>$+8+)1!2M_4W#@=RwvNq?eIl;~zXU8R?H|^|ocGP@0V)miS?d*AWOjHej zE$r-lc1ZgeZKv|IW7%0+0NdFQ?a&Q<{cJlGsU6K1>I!CuHf#M?*fDv}8LFh(*(>ch z;aV+d?d+L$td|XAXYaH_wr|qT9%_dy#{oNgsU6cEJVD(K_EZPNPW5uIw>q%%v6ael zIoM+zxYN|?-@#t%fV9j}4)$CJRB=`Z(81p8fEExVW;@t}9mxCV3w1l#iye6A{qe@O zjSkh5wR3YoclJ?*4k~H~KKkmHA^*A@Y}5|OaZNg?v>lM`alpae?Z6{a&}!JjYaqU- zR}Fi44RrS#dwI?O_4FECR;-mt4SRbHuKsAJDyV*7D`88PK z3wPZ6MGbp@4RnVXzp;iIpa!1w)JYu*Yp4ZkFy%R2C2Ob&YM?9jqsba-gBr+@IZ#85 zPy^-i)h?-}R;Y!z@m{sm47HeX;^peDrFN)=-XXcS)lx&$Lgdh>T55?}+`a0ve>**2 zOHEM=y(0N<)lys3LRXjxU(`}#)S~xQx+~UFYt*7Zu3Bo2TIdr$(N#%YMMI8aun22+tfj0l&-EiYMeSq z3rf^c>(oJ6MpZ=WsCnw3js2M&oYX!}2%_leqy}>0crkQR3pt?)NB7&D)I?5*+#Ttp zHgclt-Dhu^@|=?z$q8-sd%WeOR&wIr(YeQLpF63UocKES_LVPfa8f%tpE14^372TvkubRS)Ht z)bXvS_Ns?S`;qn3VD->6pzm|_)ME8mylLQt^>5Zwlhs3sZsoG-sm0CH@ z!bA4POI_4;W&jqnDBVE*bE+h|XnB=17bKy<7T-1IpbUQXoscIKBpbO%{ z*1M<$UASMDk9-$3p$p&2g1D#+U3g!aFmK#Njp#x@5xp*IMHfmhYOn4FYQ_c#3+mB8 z?brZCj+D%3poVO~0s#OG)RGMl+cu(snz8}PgDMBrKyBFoS&laws4*L$9Yk!yZvL$=2(H?^!AnO(sL<}Ywl z)4CxpdcB+4)(x!)lX-4xTsM?~QO4Fyt?PzJ=D3@h*A1=E(!aQ=eckxbpRIUYBQt%fQX@Cwi|L=Jw%bUp+z2@?vl^+H z8=+6j;02A;&W#XCu&$9Bx)IWH^BSq88=-f{P0mJY>PCn~h&57MH=^adGgaGdq{eQ9 zUX7dEHBoCfLCA3TCTi{`XrQ*}k|t{JCfp)d6E%1fA`VS>nyAH_gqbaVriq%o2~s|0 zHBp;4K{Z>oAWhWhP0(Cy$+{+L^(Ke}$!nr!Z^E1(>(t#OJ^i?TZ-ORZOJhyc@l8-B zEjsjilNNk_>64-fxB`SSucGrtWXX`2Kx< zy5pH<>i=d)NuAkDAJ7aJAwBDw=?j{nRqBDXb82su9-fe8Daxt z&GZY+(5mdv&&~7=&CsA}ZB`5YLkk3>cWR|$y(b&|Iq@GsktrmAuYIPnO0^k z^dl`0+Z%16FKNNt%|ENVh5n=keMFRb=uSFJ@hjkXu>pdriZ@91HC(re(IsW@jyu3S`U4W2b%hf&h^mmc%T}t z;>sTS9uLIkM?LgE9%u?P=4TImkO#-)^3V@?p#bZ$&`DnUA}=KC>E@+B@D=!`qQQ@V}@%u9dg!?hxMee`)gNLtg)N5AKTq!|5t^nE^@ z-I`KP)<^&6gQfzhK|cCGAEc$+?4uv_;l#bVs{7~*ebBCd!b3j#Lmvb)jrY+f`k(}f z;?zF+MIUZeu0B20N8jj!R`dmL_~;*fknQ}gk3P}|6*H)&!ZK)K|w5K%zX5jK8S73@zHPkuv@p#G9P`X4=U494R3FH?;WZLwyj87WL7O z`e2#*^3j+2pcP^9Pd@rnA4IbM?xRojL5}Y+AN{Hyy=6iC^sRnKh}g|f|LTXNduRLU zWBt&Isq`X0{j47&WP|wWYyA+(aI>HO)(@>39A3SPqPzzSj>~ zj!Ay{Uq5sxwQ#DRKG+YjC~x@bhyAD$P~xXA_Ctq>T|V{GANwIe(qcb-vLBK(tnt$? z`|-OJG(UZ_AA&`4{PfR$h(s#$(?|QE!`(A${q)m*Tp?M4pT61;!JrX8{k0#mJxo7+ zwjVtN)cWbS{dh(igrC0K4;^22JLaeV4nSpY%BBVA!vm1?^UMJKcmS&edsce1_l1lrw5>u;2t*z=+^`ANz)C`w+EoWjKb9c`u6~I%C+o~0DXJ_a%3h2 z=;s6ItWM1BQv>w%0qDrI*Bb%)`v9aR%?;4!2ehbE@I64kAHZ=T6QJ)8K+>T#0s8*{ z^ok7I79a-*K*#lG^6a)%&9_TLmFe+WW{WozyTl0yWcNz=3kgX9rG2&#W1NG=hC4u<

p~JJabA#j-L1+py{gWX1MGz9!E)J4o1fk~=&RG*A z&j><=M9TUG$u)vFO`1%Qd?N@QrmZgxl5+$h+oL8(-Vwx~Vg!Ta9zjSH5ebri1R*WP z2$F*Yp+J{{p+WMHAOx!%4w8!mpysi$B z&xEj5t`Ip*2-+pRy+1@=6M{}3N0f%hZ9G-qfH@loDd{R3WvyZLJ$dJ zgvfP5&@N)mk0J7%5Oj(->TrmhCj^nu|AokV!qAHT-%eq2pD<*wrE8e{Ck&Z5=^G{o z3PUUGxfh1XgTl~B;OK#2a-lG;6Jr)89|}WC>aZ|5Q5ZVu3qBAgFA77VvTC5tAT3O#T#xV2aW(IaC-rX^S|*Fu7D1T?8D2$*025N!oZL zOimSs8thT3EKFV%hG5adVREZ5?vb!8Onw!C3RaXTi;!bQAQL)WBjj0;<6|p*Bjj2U z$Z@$aLcSG&J}n&wM##A$5G-?JguE*PokZjei;#OopbVBu10v*K5lGZIE5> z^075SjuwGVno|2B*KQ zpp&A^4-xXW2n2h#MabbI5DWZoM0)yhCKrV)Nv9~;Tof|H)-_5-7lll%^^KC%MWI!@ zaY2;KE(%RTi>`{2-9;hj?Zzk>UKCoTP8t>^%Zow{9jKxjCDV&S;_7ixvb`uIzM2>% z08f8_KOfU** zFa>)I(s))<3Ky>*F^ImRHPV0~g_k1^=^ zsKpn=$RJ}-CRas7F|x=Q1g+i}Ba@6l&;E5A79*RCK{Z^Jc*e*mV~}x!u`#mB81%eU z_QV*OWehLM6(hThL86@LF*3{;^cY?DcVc9jF-QrSA0yL@L8ikN#>h5f(34b4SH;LU zV-N|lB}Ud6gLb(?_r=IOW6(pEJxgL_pD~DyvB$_jW6-Y7-WVebjX_UVEeplSL}QS6 zH4`HnjX{>#@S6hs%GzP&~M`L8BaY#u$F-~?GhYVYGiIbtmAtR7|;$*3D zh$OopPNo`%o|f!=Rh(=!4ngOG<7BLHh^@OlPSzTSp1fLlf1J!U4jC628z*~>L*nC! zaWdFA^rT>)7vp5Hap*Mnz_d7-Y#f5_=ETWn|diIt$*=-yutXHZmPKFytg@j3QvfMcIfYw@joJ=%NiI2mso zZE85NUnWl08;4E;+rE#J`Nkpf6Sz#mxdd5o0y0I~B|#>ffF3gK z-zPyfoPbWs4i89>5htKm!+m9ftT+LQMg}Lyj1v&se|v)LIDz}+N{}HZAmct`6J*H= zNc=w`L8hF5l#j^?vgHIMj-8euV@^OPJG15_$eI%n2{J!H=A3{`UoK3LJtrWec`FlS z&!CW*b`*b3CMCZCdjH2ka4D9g3LMr8E;4@ z$gUI6ihaxX2{P;iBz``WAj?ibQ1sCRnRXH)$xcj?Z6~3oY*j%`l5r;?!_K{vWZg-K zr5lhW^G-qy*C;faB>PT6PqmF4oFoHJLa@;7NwV-HG|4_XJV_>=gp6yANs^5xA>$hp zl4RsbXsZ0qn5_&>y)U+g-c@l!&=OoF_lhBm+zmJk+=t=15v%ptLvh*a>@RqW< zNiy{$G>vd_p^|G$F+)f}N~$}>EFlFMDGa8V zDWo7HM(Gr@g%q?(?eu+$8AA$s0xEeh#jGI(J;9OpPl}mC8e;1@rkOpYp(mW4JR{8v zA`KbI@115Ak%pk50cmCuY3K>2^c86wpLI$@;_DmI%qY?jv^+G;tRjtn)P&N+;b~?T zX~@Xrm^8DCG-QNoLYf&y8k)2lFQl1eq;ZYRfTx*hq#^Oy+i7MSX=t+i)JJJ%9BJq@ z{)Vs8%sSF|QHG4t%skT2B-PxMX7-VWPREP)q?v)FA#r_CnpsF18)ObQ%}gW>omLNa zr%6$M5V_i-=&$Eq#+|h2h+?>(vZ0DpENU+48-<#%rHyI z;Pf-}m0_lmfz1A&m0`A$flgP;&d)Go$v{fT6&YqN8Qdicl40hOflfPz49zfm$v}VK zqs#CNGnfoyF2f8c0~rB3m|+%_f&MB-kAE`Egfg5@ zFqjP)^h5?TB7@$@U{++%BN@z$47@5L)L?dGpr_248_bXldMATfl0gq;FjF$bb?U8O*Q@dN+evmO&3^Fw-*V*!A#7c7c`iS8T5n(GctqT&|p?( z&?6em%nW)(gV~vSZuzd0!3@oycQlx#8T60_Gc|)=(qOh`&{L|Pr{>;|=WGW3rNO+- zpwBdzyBYMG2J<)d>@wtFFo!efKMm$_27Rc(T+W~$HJHyC^rZ%KI)nbyU|wg?ry9)d z4Ej}*`JIWU#J8Hv@l5(xlX;#=A8Rt#GwEkd=6fc6t;w9vq`x(p_nGv$CUZZNe%ECF zXJWbdUXwYXN&jmy4>ak6P3D3o{jkY=(4;R`&wU*7(;u763r+fDlewWuzicu;RL?D6 znlYIpn)J^m^F)(A+GMV1(odVr7ft$V_1qGgnDo~s^G1_C+hp!&(r=s0A5C-=-)%C7 zH0i%h=8-0SxXE17q#rk#Pnz`Q>bZ{xQS|2~^GcIG-DGZQ(yyD$FHPJcgZCzLOq2fI zWS(i#$D7PGP5OD0`KC!eJpeFsl$vo5~ z2QZn7n&bf{^HGyrKs~qQq$c@*$-LAgCoq|tn&bs0^HUSMgd3R5QBCp#lX-NJqqpKD2?tv7KS`TzS^xk5 diff --git a/tests/v2/fixture/test_format_compatibility/array_10/compressor_6/1 b/tests/v2/fixture/test_format_compatibility/array_10/compressor_6/1 deleted file mode 100644 index 4f8a05c565e50ed765ce68acae09b093bf2eaa49..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10775 zcmaKydDxHT_Ql@}QG}u>Bq4<~AeC5^p;9@W&?J;lA`Q}DR#7sfBbtZ`QHB&vsFcb% zNh(Sq)icjBM27tKUhn?<_gr_nuJ^BfJg-Vg~pdPyU)R6H~lvM)dw7OU-)9>nnN9& zb>R93COz!nQ0K~9jvD4* zUDRGMOYVq^eE$ve`k(LOGpS-*OBd5$`tN=JZtdc(c=hlDjl|!9flzZS>IWB(U3vuyQ^-sUO^?Dbh`aas~ z`))3-U%@YfM*J{Rx0Z3Mi}i2xlH+3b{+j34z1=OnrgymL^wQJPx-O(??-+TPi>%Gz zb-V6%v4$7b#dF*5mE+>h=GrkX+Km3A_qqpNRAIxokfv?=?xQaFI!3sd@Yz>?%^KxG zySDmRsaxCSaTjG*>~8$eI2YP2L!NRWO)CS$#mG16-P~)k3%#X3JmaF_RsWIWV*iU* zKmGA@F7$S^c;3Y;Z9bOPb-~Xv(}jMO=U#Dfa`SlnF|WE{+rI8X-^QMKE~cG!(gy<; zNZt2;E3NB7tLV1a#c|aykN&&V#qW0 zn_P4_?I$@dYTZBe&FMe8;5FUm;*%4~O6$6irro;tHy5R2%*92#s8aV0VdWb_7dpkh zj$ORF@0Kl>W-gv&!?@7t(R7y!zK%UEj%su1y(jIHy4koc*e-`$3_p9&oQM8#q1}>} zc;GlK>*23@I>|ifExo3qhgoZmmE++)4l)lNY?WH3#4^D@UztLpkJkOO%Kn` z?{j4A$sX9YwLR$D7<8(K${g$-IMy3@kfuGfzLAIV6F<8B@-sc$Oo8!mHXjcqqx}E8 zc8-Tl8}z1n;Pq(fVK@u%&}GfnXFYO}2ab<69`^QWC&$A&z7P+Tm-Zg+?S8kku7~#X z^~-oTi66#;s!M}wJ*p_~f+u)l$aGLgzx_MDO zjN5RV91oPaJ3KVI?}zI??C0Ugr`j+cbb8Fa+XG+6JsyU=`#~_}UJu%}dB-_8^V>V8>TH{8P5YFCWeE@I6Jw z!#qA7L~twBUf|)HEA+NX?G!r?s{F&3die694zKN8=7A$*xd&C;t?zjd!IhWdffBRQ zgDPE{k3FoU_?i+K?GNZnFqZgvr2qaoYz>| z*9SjMc^~>^4p;E;*z;XQc73pQEBg?^UD>m$k3$o5xclH}ui--)SKN>fbwVvp_Ce{d z?c-EFKGcCsJI%)zpXhD%!7I|phbsT>#y;@Qs#69u@xc+&%!ewj9Ejk`3-LjTY3W0i zZutd1T2lOd(1Nt_p$b!@osT6GE|ud$1XoOm51s4}cJy)P2z6yr_xeU1dw2Gs6Lr%y zK3?Nx_3_Xr|CZx}QrFE#U79i!I3)1NAC$b&3x!BU2(q;O74R`L~v!8`Ow>O;-fy)p-vp`gP&!T5B(|+ zjq#z*FnP=eTX&q)y{ui!AD{H0j_m5EeQ>l-@*#~YZpeo^p|_s(LFs?a#|%C`)PYob z(Fd*J%RYELX8BO%A2i2@80BfdzvhD@WS$RI-1Tqx5Wy9f>Vp!qNa_})Yqr$KB8tBc zT9D;FRADB)=i`hrAIkATi@MT>PWE)Qk5~7&(z-sRX;=UBnGcS#FMQPIMfH)l{+}El zl)Cjkrqh)97{G?{K`XRL>ekosqYtN!nnNG7%3FP~UAFt^S+$GUWgprt-~8c29eaDv zhZtp8~81DxU!AK%O{00Igy50K6V21)%cR z3e+f9x}|mij*q$lRB=0;7C;17TxtMHOrrp*bn_YqXh`u7Knv0=fGSM+a{|mBcAlI7 zw5Tlt=w$DAL4cF*QCAj#D@>aJ9Aj+*OyfljaOc~X%LzcK>lmO4O<927*)USKTA^zK z=mvAJOMo6UhXL5NT?6Rm(Bp;xKR)@T*yR9RrFsNV$G);>05Qrk$)s+%rStm)P^VnG zUjPwY8Ds(Uc8FbuI@A$^0`Rj84xnG9|NT<8TJ|9U*tQP^5W$t@Aplpp;Q=_-M+VR; z8jlGeMp>4a0F?f50iNU|b&CTzG$8=3;iLe(9#aCS^7ni;09S`;0XRNp1W?6Y{Xzf{ zTyd!ZC@-@DsM6J)6JQp_KL9Prya1{$Bi{&6bN50y0ccSd1<A$Jl!Tbfc4tiXCUJloNn5w>p4sQq?~VAa_GrHvp~B+5mhVUj^7ia~ObDd3^xg z95!tTAPcT+bpg0a{SZJMd&?iC?p=4wBnzOowBFVL>Xe_`9zXG9qWGu&?;u{4Ohgl?B#pA~{@OtTOiWz9qAR@d~r5L|OxgrL-2 z5JHw*F=ipYXVZkB9cmjwx0nKdt=LnMUk{4&i# zaNX`2Lfx_)h~UaF3!yh;-z_23rQXvs1V2sh5c*|q>=S~vy>AG%?wuh-aAkoA!L@Es z2#)r_A*6A|4Ta#^KSb)L$b@)=PY87&*{~3rg4{7O1h2^G5UTuL9t)xA!{<+g;0Sp# zgevZm2_barZ!jqYC1y$pRk|wAhG6_KEd(vd^bo2rgI@^2E%zmB!n!vVwt6GHcbdxp%o!?i|PA8h%aakL(nd- z4x!t_+D}511|d>6jcN#W?Tx+)@u8#BECko>^&!+PkJ}JJ1XqSx2*xo#grG~^9D<)^ zi`31b8iKZcdkD7fju0Zak_?3?UF$-Wj`k2*#ndPS*ZwpFr9Tg`lTQf7Lc2p~3bJ)? z2wso$ay2cS?$rZaCfiXza z2(&`YBj^^h{oDwQ$y!99Rc;kQw}-YDMvx^}>~aLIQ*9%tW1ocxVw827f$MgM2$bAT z5j3qEa#aMzF;_=WhuW)))J@YKfpJ~e2(;|oBWOC+;-&~9xRMM-;9A!+f~G>#dPmSI zj_DJDYk%Jel>Yt^2Jnf%SZH7bO+mU1j=<}2Uj)X6Ln3JUuzaZ0&BZQ)D(>uI5p?UX zF){)rW^@Eqx+7yFFn)L<0xih+2rOel1jbVnBhaEwiNI?*H3H+yX%V=>Opm}Z_Cf^7 z$mF6%;F>#2>gM7fL6%&x%MloZ%#A=RG(UoFFx?kMU`+N_1X|@K5p;7{@pc4FFPpv_ zfvePt2hFc1BRet+_jbZvB(@MxeaxkDyBT@PP=79}Y#J1vwIdWgHcQ@l?4Ow5UhN z;59ua2II?0F}T81iNP^eErxD%$5oG^sq*6|#-P;Ij3G;|*yR{ZbL+&Q6{;6QH<%6e zV`z%nzF`bnrSy#^5T|Jcc^ex5?-WDRsfJg@V4Qn(46bxtVsNZqCv`I@i@~+Odkjkd zjWKTF6N9nPtuZtO>DW64ugC2%7#sGDq3OfC{xLP3BV<4fM#}?Z=;mL3a12V!eK90h zmiCRo2x4dq+K`8(c7~}j7*UOgp-Fq2(J^>c$HriU`9us&xL+P0L(Ouv2{Cl58$U4y z*WAf5D0Nd~$dW6TIfe$yJ*LN?9eO^7ZZVx+ilIsBs+lopmuJV&?P30_F&L%Jjlp$l zehhW(6&A)|)c#fsuG@=as9V1C?HHQaZF?sMBbgO3Rxuokp-I^#AH`r~w<-p0`zJ9p zp_;WO1|!|IF}T)!DRt{;Ul&8G5bqAI{oll(^lyyuJ)anihBn906lC>JF?dCOiNWY_ zTa4Y3KGfY2LkY;V-(!?U%Q1B8ALYkT@~}IMLG+NsVEm9t-TW{ySjMgxjHvd;piSKu zgID!H3`UrTVrar${YVU*s1uG#z=*qSf+h@?6Hw|ZCLjW-lz=hFaS3RJswL1Z=93c= zXp-9U#00dI97QB2bWjAPDD zKudOR0!_+RoS&eS2_?pa5@SM%F(Ij&!#y!3lz>&VPmBpA#)J}MLWwb<#F$WGOi1cx zA&D`e1f`2zVoWG8CX^TxN{k65#)J}MLJ5eb1}4UY5@SM%F`>kmP-09dF(xE+^P(ok zgc4Bd#wNyu5@SM%F`>kmP-09dF(#Ct9hp#KOeirXlo%6Aj0q*igc4&yQa6WLVoWFj zE!l#^m{4L&C^06K7!yj22_?pa5)37bNsI|4;M)IjVoWG8CX^TxN{k6f-4wgTm{0;u zAF6Cjj0q*igc4&yi7}zXm{4L&D8V8!p~RR_g3@>@F(#B46H1H;CB}rLZknOQm{0<) zx%(1hLWwb<#F$WGOei%blo}IC;jnR2V?rqyr5=|W6H1KQdsWBm`o6A&cOeh6o z!%nF&AyHH=b*V9-)R<6eOei%bBz05#Q)5CYh^B5$jR~d3gi>QdsWBm`n+tbpOeh5- z?t4;WLa8yK)R<6eOei%blo}I~x@itmV?rs2P#;f?38lt_Qe#4?F`?9$P-;vl#Xb(Q z)R<6eOei%blo}ICjR~d3grsgR52-Ps6ddbsrpAO)V?wDhq12dAYD_3KCX}K%g)TKF zlo}ICjR~d3gi>QdsWBm`o6BWNO)up_DKzzO^?hnwC^asW8W&293#G<|QsY7?`q2=j z#)VRpuDPjkq13ogYFsEaE+n-xgh-7GrBIqX@$b~QP-X2_ix7s~J`4Ps_oC^IgU85hco3uVTIGUGy0Hv^f> zxKIY7m~okLq0G2YW?U#UE+lmmf@H>pGQ7+uP-aXhGbWT76UvMUN!<)}Gh;#- zh_^Ro#)L9sLYXn4%$QJSOeixZl;Knc5ScNd%$QJSOeixZlo=Dsj0s8I4E-}>LK*lo zjDwjmq0E?2ZcHdQCX^c!%8d!-=+1yDHzt%D6UvPV<;H|^V?wzxq1>2IZcHeLrXUp? z<@V?wzxq1>2IZcHdQCX{0mL(JTmP!7fq7v;u;a$`cdF`?X;kkm~klp7PuagYml zZcHdQCX^c!%8d!-#)NWXLODKX$ekM#%ArZ>=-YB*Lb)-a+?Y^qOi1b`Sk8?J# zOm0jlHzt%D6UvPV<;H|^V?sF&kO}3+gmN&_eKI#Dlp7PujS1z(grsi%@E|uPlw$w` zyWE&iZcHdQCX^c!%8d!-#)NX5M<$dT6Uw2#erUEdHzt%D6UvPV<;H}hZkmwXm{5*6 z40Lm2Lb)-a+?Y^qOei-dlp7P0y2*rcV?sIjI)2QJ3FXFwa$`cdF(Ij&Oei-dl;bBx z?YS|b+?Y^qOei-dlp7PujR{HJWJ0+yp&UfHhjU{>g)yPRm{4I%s4yl}7!xW`g-ob0 zCR7*`DvSvg#)JxELWMD*!kAEDOsK#}2AG90p~9F@VN9qnCR7*`DvSvgSX!o}WY>`+ zlRJ_P6~=}NV?%|pp~BcuVQi={HYD}aoE64~3M^;PT^JiGj13jWh6-atg|VT+*ieBH zWJ86qp#n-U=l3a$4Hd?Q3S&csv7y4)P+@GSKtdQ&7#k{#4Hd?Q3S&csv7y4)P=R;( z1BAlZPyzjQ%ApB`v7y4)P+@GSFg7H03kkVmMqzBI!0CJ;g|VT+*id0?s4zBE7#k{# z4HbxKL<(a=1@u>kU6&Qch6-atg|VT+*pSrC<-agCRN!W^p~BcuVQi={HdGiJDvS*k z#)b;i=MR_)V?zZPw{Izo4Hd?Q3S&csu_39O0a0OWsK8XRp~BcuVQi={HdGiJDvS*k v#)hPBvZ2D*kkrj*r~LoelB#7&j_6bOI{xtl`L|xlQKf%7a^&Ox@2~#{HaoEb diff --git a/tests/v2/fixture/test_format_compatibility/array_11/.zgroup b/tests/v2/fixture/test_format_compatibility/array_11/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_11/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_0/.zarray b/tests/v2/fixture/test_format_compatibility/array_11/compressor_0/.zarray deleted file mode 100644 index 4c38043a12..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_11/compressor_0/.zarray +++ /dev/null @@ -1,14 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": null, - "dtype": "axqZsjQzKOm}53k3T~I+sokzwqx5LTNO8R5@PW9{l0o zgr+Jh#0P9?LY(tsNC$=@UCEI$$P(1RtTAi-4`Fln3Z<81iM{~mNvB%Ko1{I=8^SAY z0~wi=4m!!lkjPcpS62k+TIWdanDmbDjGF771^z6*PJTBE#bzjyw84wfIiZ2S$b8hZ zi>p-s77#5ToZxcB$z*Wq(ZfD-8d!y^VN0@|?_@pXQb9Tw+B=ujQ+HbLa#!rJ!V{|i%)`fh1!5|dK~EO0F>}=s z{*%r`w0-bn=}Ypa zmO-}=qqMTnIp0UpQfjR$jT|OByQ`*0IDs<`tPTy)EV4g5id!o$0|orI+HB!Am}oc2 zfBFydsj>q{s?F0{JjhZwz`$QZzv|0q2i4Hrsb;0!Cj)s$_^!~+$Y+wwgH%5QqBq%F zS~SxsFf2Ki{YHIe^oA!vzU>H}&MaaRxJ$-7l#7n58JtN)FnQ8v{3;(II@&53U>(&+1UOi~AU2D}b1)+Ob+tB0`^ zXTe+Gw606{po{LVM+ytbLw=Ag77g=vlc(Bup^Ky$*Vp+Kf7SGnO<)M!ms5^|M^R?8A^OEVizYZ-c+unvj_a;x7mh(q8u`!h=AyFrRI~l{wOl zE`;?Sv*)7|OoFG@I>hm14Evi_?u)lyQlhMd!NaD&&kJ<{H<9eUU`OEv;dM-{eKh9j!8*?mdcf1@jvOAiy8uF zY9#!OK2mnynoGY$J{Cz(7Mh`tlYaz56%1uQnx6z8hHk<<-i5!}-3J+=577s_6>dU> z$aPNVKlRV>&%rt7Y56bQPp-xkw}#j`My;pbxWba4l4ZhI*TA++#xh@dltdr;B5_n-Zqpfgz?p*^^M`}|$O7es(F1IAUvb6SZrTm7AADHRjNr3+}T%kOMO%6?_`hzPe#jq7vjaOQiz*;aG zf1r&r1p06JTRt9qC7fR|5UC$-MY#@_O&^f~`s zJ;K}>9~Uk{(~Oc}RlqBbloycaZh)6LqPgEZqoFCW%@B=s$zVF*Wy$)A~^VAW67S4<4YS-rWYSa=Oc4TuO zCDWec+qI;3qtzDVJ-jltfJC_mRHmqHY^}Hv#wxJhTmk>(62MM#P+F?t zF&lB~)d%`9uCsOCItepCKAa+-11qSR^k-CO$2B!V89=6m^FX9yd`vd88%K~;$^xmJ zq*W%vb>MFc5tiznmTzt{^XQA<6WvSWNEjdx}@t%eHkD()OXr<2pixq+yuQssodxgZ_RO^ zMISn!n2k~^QJVF>WPlsq*WwO$q)^UIa60)*>U4!=H|jsDF-^Y#D@j{nlrT^zz{jnR zxq8k9U>k}OIMC5s#&luUVGma%SH%Bh7NeL@g49K*56jF_+zCb4t|m{AR-;I3H@UsP z3Y95+^}oabY^s_3GI*RBN`G!&Mec!Fo@HHaWKPd};$N_FKe|^Exlf*$m!x^Z1#cgH zGt2@z-9N!EL2}PCfd(?Q+oJw)eQNEX7Q+g&8>C_dXaB(+pv^cX zrF0)IP?G34Nm>UVpvJJJcd%~@GnX6eo6Agwy`|;wd+@s%t);7l)NSE4$tT;*0n$-w z6ltepFh`lJdT>WVKtQfBc1dUSe%wB3pUa{DqmIm2-#=Va*&p;;kNFb3!Iv%^p+56&gJ&x@@y(T% z@&slC>U6oGV z>FcA0>C2>%5=oDBmytxS8NVd7j%g-pTvqu=I^S``vOBBQz37>zzVr?Gg1#)hH2;_U z(AL3h%M^l0`9&}yEhZeK511!XQeaE2m2W$a!iKv8TW4aN7|=R@RX4U9c*nsB@C271 zEb2L2IqkhDpY$|grg(F#DBRKq(M>o+&g!|=HW_K)Y#<2^w=S0EK~v6lKO{wDnz8~v zaSCb?)f)6BPsJ*f?pY{H6{@j|{3o0|m@7{8B;vnFf^!2{Zshm+L|SRBbPa?5*t~oj zS5s0Ss!|^s2@%u+VYJpAcNUg_3}ZXw{h!F6a_Q!y^6NoAQN8<=Q|dD9mUK}apSZlV zDQuTwOYW3XjkgA(gi3#kV~oSg%;(no2N~7mO^_fs`6407Ul09cj3ZaUZ#oBF_@|X%gK4XFEpdPsqCW*rNJN@w#CQMb*o#i^OXkoP4a410GZAeaxK>iZwoFDb4CX( zi;KW7$(qn{c+NeFS}MAX!B$P34c3Ml6F(eiy2E>@8wq0FWxXZD$WH1e+;8uLw%|g3 zj_)FPOP{N*grAV5LY5~+n~k2LnP{sVLKW~S2~tDKz98|&+p@jalGLNA8>Cb+M*GNL zV~sHEAWA3}H8@A_8Ja7807ij4xsnm&+(4VqErAWqcI+4J^p~DrLqp5wch5F=kfV4C zx$C_IqiH99ko#I&Lv;sn>`eDHv08o^wx`V6k=|ZsSh+ioiYkKbTAN#(HtHc?-Nv_w{!Y+Hu3Afzn?3IUWf&m5l&H^k?d1 zce(d1#0s|3ugpYHi#8;TFUxf0gfrE}ptbb3ktw#ruhgFEzo`)bP)&TMI1N86`&R_q zJiUeWoO;hX8_?_8U-NLTvysMC{4dYt=`cisXOD zhx{70wQ*g_;9A3(p^+|J5=&CZBjHl0G5y;Am{k=&^$?{WzQk79qnT+2hMVAYC0{C4 z*OQUl2U0WhH}`&~8@Mi=6x&wZ0BxyEa28~eal%FaGeIvS56j4hyn8H)EP@n8wu&s2~vJf*>+t~=!bER{EE+5AR0 z-Q2^(@_o!dm2==MH(WajPrJW2Hj;p0G>93b!F zRB;)Yj2%6MmYu1#3*48!L(7xwzky{~+nyo3oD8TL&TqX?o);tM#tlSVy+ z*U)t&#-0zmFt6P$Jp=taa5?vv+K(BC>Qlwo1P~75Th%pd!$_stgDHU+@`%NFzUHQ8 zNoRWe9Dj$OlNils09jpgWHr75h8>!&mV$*|TezVWWCoO7l&+)4N?Rilt_>w?`wnON zhImH0-@wt~_q~mn8eD`&%CSm1Kg$?YnxlS*c7ZR=vs8t;mnl>q2+NG!Ap2m3Ig8H> zT(*YMjX|8$QafOupZHW-i1T$XK4bpv`bZudXlcA~=v;(rwpO&WDlNaR-hi0e=6_5D!Z1@ShAs<|RZ8Z%$7kIrcIxP0eP?KQvF{{?!JYC*!5u%kC>qY&c?7>w>}&6y?U zS+hM8t20IwSBxhqNyabK2x)eRvHdJqb(|&48=$NB3*9%>Vf-b0BCH{oX`XK-o-MnE z8^Ffo^TT(7gFL;&BjRfIruvam<8R~?m~(9oD3x%X`L;CGa|Sdq(}nh+K)b{a(PQMR z^eAII55Q<=8CzJkiu~fUaUFag3GZlvdO+yl+N5;@pP>t=q%t1@DL%Z@Xv7wUqv6*e z+OwaUYIu#k_F^jEeN4ExtCjQlMbO|ZmEBkdDYm5CrZiE z3*|0Y?HAOHq_3#iaBFGk#Wq~i zz$_OdF46b*?4lG{BgHPBS6C4V_ShrkM7V4(9m+u8hORRQh4#Vq6r=u+kGDSzEfawj zWj4~chw7Q@?P;#xsr4OKqvwZTYoA)FI7S>RGkl6%nDR{S?7446yH=*0(r&PQ6Kc>8 z2QGpi9S!(=1+Z7hYki%!lerJqr+eAw2xDpJhgk$tBdLeAZzkp{n z5xng@4RQpPeD1v`{zi1BQAxwlAzKia5pP!%nv6Q8QG5$yr}V&>0k@D2)B@?15gX0~ zto$N0p7pqf;0@+|(Qv#;J%_pYZa6(yDa9J?tY^$q&IdQDJXPzf2dX_+tb4#r{Zcy3 zS0zOWLp3KZ5-M?b{zlJ|@D@0ZF4o#t4v?lfI#I<6P3ynnJ0L zg*~S}!au?r{C9My?nBPDL{9`0!$oQZAdB4O62f`>dnBKlA#D`yfQ7&bPjPpwVzf?5 z&=!Omdy2Uj;U{A?6-%wd`Tly=7wY$7E!yo4nTd)A9rG`gstryD{L_?e4rFXBTLETT z8&IE6I`=!c?r%iS8NG0tKn3$JD)F%3oXNgKnQiKI3}G^o5Du^ zTz7Z&R(B_Sm+Q|DCX4thzV~~yHKu5>B!e_zDpZ#_&a;?I0DqW6**BC(l5-2 z)2-jA2IiZcP41BM6zV6g&_k~FXJm`~fiW9j{DvC{0+$~)p`^|~I`e?8HGFTKI TE-%sAbC0+h{(ngZ^)2;(8do}Y diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_0/1 b/tests/v2/fixture/test_format_compatibility/array_11/compressor_0/1 deleted file mode 100644 index b03ff4972b80891e1780513845f05f1f070172cf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6000 zcmeI!>32=%9tUt-nna_T7(+B7K_bb?8TLN=dG?%8At-W7qtYuy%#_?VN@7fl8deh* z*PNI_WH{#}d*07}&IwUN%wq1MRm6Oi7RBXyegB4hUwvPH)^GjR_kBIs{|l6{!!d6azp7JCmY4P@$~@j|;z-;e##>ZObVLA7!EFmfJ0o+7vsZBXg( zy&IE5sn1OtEX0}C$6_b#0&L)S1eUN4Si~FXBzI2-?ACn=bV|Q0KYu`T0%@E(L<$kV zboEsykv}U>ldhwj(mm=z)Fg~Xy`p9qbwq@v(;Y|nOiSh{SD;`V_aawcd<9LQ<@u}TiV1auL8L+NZg z*1(2zrg}ckkzUg&O2GBX1$OS%KZaxQq@Js|IQ6W45;BhG@KN01fX2p(FrP=K~+P5L#wz^bdM;Rb8%4FjF&tFbM`Vj|x@Mte^Qfo)}nZ$X)%9j|N~ zmHAEg-BMz2a-gzw8MK@5tLPX*># zGf|;!G?`QprGzO4Jmt)k6S)(@Yh(kq)B<8ca<7U@R9pHLI#RYWP+J4dnPe%B&`f5k zUgD1;eA-jrMg6wG;EEJcx$fAc{YOhPPJunvIr1NYY@&gVux7z^{c`w=#?F)@fq`-Z zo-Ejks5%7F&j|qwkrsg)z;{A`{Fx)?M}#6)p^}58@)*J0HLWQA@*ZfN=m@Z^)qPxUX z^rI97yP^5ujrA{fq#lW|wM*OKpP)@*+tw(EcTKZ53V)|>5b#tx+}YMbFUS zzD2$F*X?4{0+rt3+;s7jY+C)e2lP4hp=RlYa(i;Q7NSPOM^Ld9gp)KMHr!LX;+} z3;ThUZ$vlZBoQ6#~D#jWwD~GD_$Z}W2r85OI$z+<*Favf* z5%M`}?7OCO8MTmVVuB69Jfi<&CkTgXOR7W6vS)aFZ9)mh4H=AOa`4a8RM zDDwtv$1!MvXEe5n>*&Uv1@dw2s{aw*A`IcTXaksb;4P@NSii~M_hau3`3p6eoS~0$ ze`j}ztE`26o7~-`F;);akqooiP$&7aeJeH&u@| zSfCy(dZX%Pu+?i04V*OY%sI9ThugfUhM>s?%IUcfg47JRf+G_grU6wBwtpK zdxQ0nLM(WrhF6wzdzlSZgPCH4s0X#r<9>sXVdRzXeng!13Kg*$dJGJl3cvBT~xM8(tjVR|}p~*cG z++>R=!FY+Y&FN?$oIr1Pbg+60&4 zP{po`CJS{vxq6=WNOBpk)Lv(+sY-5|ycTY#cvY8$%0V=+S?ydwVD`TA5QD}0I%|pS zRBE{u?%IM>lhSeg8*-I$Nej0+FEKs^A#7b8NM;XY+0g3s%Ip`MxX zDh1=H&ZTwC3B1agVPrCk;8D1qU&6NaIo6*7S@zv9FR)BFov;88H&X*yXpOf$zt;V= zYbDxbywuC|QhB;GQy!;h*n{Qqcyio4PX;!%!(cUffn4%yzzZM4&)C&uzD~jXs$h4q zIhRsAlU%t~UrBY;B4sRITOqlE{5A|pHUU+ctQ-3HaLc0GHC~O zL5-}=MJ+79^Mc;ppNmY>Nt=$p3`rBY1P+Tz>n-b@Uyc@Ga_=Tt=` zP38wMy(xX;vmG8+q5Bg5MBGDs64+)6LI9p|f95-Y6n=wo0TR~DnkUSEYj4T9(V12R zR$-m87tUiF*aEy>9|mTE#bST6lKqn`!s9g;aR&DZg!+3Eb?P(Cqt#f$)It6Ldx+l~ zSgOud*85tqd1_hJpD9tEFrhV=rPb@XB}>s=6));|0S%XjV@>B9$E8+ms)?OQ2{P|G`@HAe8>3ceY8R16(Mk{_gjPh=D%IAe zR&9~w=8iMo_dV}9_r@x3lTlkFgS^u z>4~I98CmLk_yIEquhqMmg71(ctFkBcb4e_2oX*7LQ zIbPl#{Ndk(rYbAM2W)9VobzKy2ZkbD$&oV164bz~F>Cz~VRQEirI%!hz5wS*r&`FH zq&>_V!YghA8JUz0I?2b7$W_@_R|M!<=Sc3D^p5b1n(Lkg{w%*vem4rmW+;-h!Hdy3 zp@F~1eAKgxt5p9M5G@~^;Bv*uWN_-y!#;BwScR)$OR}BsWIg0kK{^-Y-x9VMrZisn zlnqmVWrxB6$qz{hn824SOY~lR394`34~p;|x&i`E%rRezeq-9|)Ry zh&YJ;TyE%Y>#o-G%;|79-`=q;u#l@Um)aMA$=v7sBpH+m=m@9)h1}czgHfH#0+0dR z=0l;7X-6)RS~^o3;NB%}U~SxR>4fl_{mJOsJD1c`cUtdqSM0IE6RQBs!^eFEVk(tE zPZq8*bJY?4lg>ornYfcWm3$odDz5k|^&dR;c)zI9jW~JRH19?aIuF%cMXOhf= zR6hfvH`!ZSG}9?CEIF3_Mtx@Vh9^P3?FgREEMgP5OU68ui;k-qoJmD6dD3V6Djy*_ zvMYOp*9Sc4S5#0ID`v`@0=VQCa>TvO^nst$GjJBU=!tgCiEl5`=<9Y&QU`bjybdtd zCFQxRhp`lA!CT<8u1oizi|(#R3Jb_XevmB|4fA)Cr`mU+i=-LX*ZCEH)%1}~Ulvs}OI!;ma2wyuzGgTLCEkeLeNF9;FRUiT-$gFv+~ zpKZaFIns?Tg!LY?=c5x$f~VFx#PMYe`8D>9VN8ttGbxf^)B3{n9G4Pr%P-js-$B%u z>TOgy&anRrG(%b8^`Lz-WCk*Hq<&q zw^u%qpma@I1fRNh(4Wc;;de?7bcmUWG(S zCm%?g^ap%Qg+{(OTR@v7D-SRnxT)Ae*{An3}rr=p9CLR|2N|Ic z(FeR0ZbF5~bx!9$_0RCn!8zt>`7hj0uErF%hS)hqt*74Pmgt(5PmNeGB3H->-0}M!TEmVevdYf(BV?zD$2B0~oi(|N(5~kkePa{7!TJz|=L{_Yn z7s;c=8Y{y+M(m0&^L3=<`kIB<%MS)Ly+IMfzCb^&je2tbLw^tTm*h<4gp?be0H5Oe zt_R6Q@;SE+cW6DpnS&eihlBmd0_BC#18j3&amCth+6}NDd|1(p;`ue?XYfES0WZkS zP$?gUw@}^XG}Kg^PC3lhIJ3ujb`Wzsn7OYYD%eV!N=2}9S~paM=cXX=m8Ul! zW!AV~?o;r4#4qe~Eu-V5JX|RCgd;rPbD2hq@B-mKwbmCWGy}ImkSNj~_*qydwa&}N z-twvRIsaWf!rU1j7cN56jFMngz$=cF7m(*}fR{OLhLDJx>FtACNejH(aUQg#ZQ>pP}X}6)|zGF@b z9kg}XJUIAR>}zyNKB~qUf0kYpWY1o%QDx1c?8KtFr0Yw486Y>*ciMLd8{jb91ieD3 z+~^Q*&2gVaA3C3yjZ!O7n)SY9fE(V|;tqGDP|i+pI{8cLbcJO%>OZS7O}_yvNn2r* zFi!58;TM*(9v7QbYa$E4_72t#Q$U#qnJ>F)J3Qd%gj>T2}RhhCQp!7 zqeyEvxxK#%l_`Dozr+A+s+s&Uc$^tZe{Nqz?txjJWnFD#PS1PdU$Coh4*dh5gwKVG z+6?A9bWGR=rgJydWdXsxLv?{^^!w^W^SJYJir}unn?0?W=1d4Sq1UmC#VTXBbdtW# zy&0Tt97=xWSZ%&9JQuH^cI2uO(?d|slt#N2nDx!sm8--v-5g*I@${&67DLv0DQ}fJ zYf;=0{a@o<_bzL#G9_i3oGkjRJzFQmzZ-nQ?KM|vCG1T8qpjpPo9{U zqzKfy0Sa?dk?1~RqVqW*DxYVDvF!wRz-q+l#aBg61Y*HH5vbjTVn zUR2ud8zFv&UMT_rB-6O5_tJJ!aYh&VD7#RfiD#!Q3bb$ykS8a1L2>d(ot#@X{Tc_N13d8a7RKwK&~-%NoVwa+&*ca&#g`sW|7TcoqZR)3_mvqL0)l_ zizRE>%Yh;MTwEWVv8BpYp5^+l+RflGZm7hgj?7r!KU`DUAM{#}`4YUrmo6NkKJ#va zXDc`H&6Sq&1ZD*63$}qbf+xT%*x9j;>n}X#hO61&--!XJ0%cPMKNso1xkwmT;za+GdX(no1R{2Oe-*LpUJFC^b=$WU!^bPrf zzAU{o|CjvG*1>Gc6oN?kMKB^QCLE*>m?u(FU`wu*Z##~{hPwk>XJVWf&^murH?|vi z$H58k1eYHy>N#9F?Y$_U^fY0ncyp{M+|mcpO*ll(>bcf78EN2bAPEk)E|%s&Q_glj zBt>MJvI0MG3ThG68uTVl#VVBUStv^ts{&ipU9tb>E@&I>p?$Jz5A3?>N4$? zbWt6jxV*F}Y?oq7?vzrEw+5nwN`H!DjKj;!=hpiN8P()XkRUkuA|c6N5B+3}BUiz1 zItO0(r=p2Og=Nwx0z4Sk6At*|)Mzu^yNBR9LPjA|t_QxCXQOd;xpIjzH1%)$w30r_ z(4iApoe2&SDbffO1QuDD%g?k?rI^;3E6F%KtJzx<1!qm9Y??UBIWPk zIei?v#s8jpxpaj0ocNt_P3+^%C%`n_G5$s3W$T@^crjYJqg3;G{Ev=xaYfuS>!G^I zy25>g23lkBN&DJ>pIo$zFlpzz7u@*ICXT78R|M~0uaQdGbCg_g zMO)|X*xf_A2}f(o$$7jlG^4z!?4t{%!5|y9#mCWgt6Q)0l?L}s@@iH9na&k*E!PTf z3oZ|HMh7j6i@-0*n$U50&OM4+D!PopR!yA^)`l7rKOAVf!+WS331Z!4y(PrRPU>JXsaAT74RtuQbWqVAo0fAvc1=m)T5~z zq*O9S`^aBojWFyWN+=dJI7japnk#+)Mu9xJk`d(GK%3Alfep=e>=*6ym!4lkL(AuP z&o+0Eqj(Cr>%9b{X(xY>`&wH=bq8_mO!qahT7DU}r`ptdzFBxQ140E%GLC=^*k&m# zIASE0cNV{+&olX8v34JB<4*dDz&-jum`n4-55al@p6(q8&G9tk&< zjQ~URXX<2kx%Vx^3bxX(%tTO&HYAKM%XH<0Gu6eQwe+`aVR|MQVy@mChde1r=(CgY?^Kh=SliJ=KPhZ!M>kn*ipt#f!d>8&BP6Su1T7HWp z%88svn(~ivks8O$#G6O~JKy~r9O5sF9eEdP7Z;d0&eZZ2DLHx~w;K0@BDyO??8(+^ z)kmF*kOH#-q;ZmqE{o4PSRTV$=5Tzf!#8%m(nP~=w zo8WXMUn*7Clabs9QZw^6_kN}uxGtR(+g98FZK+If7G#of!bSfydmH=Ol=gC=`I|*T z{ec~e_AwRhj30!zeA|Nsu48mG+56{w3ts(_mJVvsddb8S3V#m;v}Qk8EO8^&k=v& z<3O4mAn)Q-aT%D59w81*v!c!ah^maF9x!i0OaIgM(kvIdOA#nrdk(5ARPK**LH?1= z;Z}09;8eY*(A=((Mea-Dg@!{;^^_WIAguY8=mgb5fK_DvO|2%<~k z3pl%zMm>br&~+rno)5b)uiY&@1N}R2Iro>^j~R&SQ^nW>5Dwy7)irCwNTu3?DS;UB zh{brm=B8#zXL|e`e}|uw7|mw@SzU8vHNFCd9h$C|f`wjNxS>NFEz#X}oagT!epzv>0zz2sMal zp4i1K_pAxFvfby}qk&0z=m3+a4592~51waMpdOWfD(}E5yOz*cvZt*99{sldNFY`- zbyu*AyURU6yR}$$621hgxgtjzGheTd&S>?xeCJW^HNV#X1$vWeLBf`>qc>`!5aS9M zjP7d9nI+~~vpo~5Ge#9xj3+5c#xK+eX?BRQ{VZ5@oF&W~psV-`-8a=?{3U!MtRa_a zo^K_dExU#rz{ceB!*_y%JiWvt;%fG$`jJxOZ{!r1b8QYNm2jQ;wlvjq1~f6#h4!F8 zyTlIBW8|y!C}TViz-VU~TUfS={Nl559ef`N?`VR0KK*dygcxNI*S%0SY3~9X|CU?^&MBE=Z9ZwpIWInMjR_Me2QF{@=WgRxo<_gR;HZN zZm@k5YS0e{E`lE&4fuQouvf@yeVwsZH`;v@mC6&DCwRK2 zFEf>YSINRzj^XlSe5J*PgxKf)XQcXX)kL(aBDPXrUgMQQ~gi`?WA!g>6AB%higZ4~Z+g}@0< zad)g@v`$LU7K9pmin$o!Cu21gORdBC{(9CI>i1$T+U*XRiHZju^DmUD4NeFA)0Axv zWNa*30cKhoP@hmb_dB@mZ$!=;mC6swU~w#Z%G^?(pn+-*Jxq*LE|Axv%}A!k9cb%6 ztv}@wkA4A$7%Lbr-BB9h+=KJ1jq2r4V^pCmL+>Yi8I;wl;Z(CbpD&CHEyIIA2z9GC zCY@57!bbjFcX#$ycPD(8>(37+i})+P_j|N8rf9JwgEV0(RF^r35y=9`^O?vV2o>L;$yMwzRW05=&m787w>WFzzW3x|I+>df^BFBJNJ z2v?aQt#@5#3ow;5T}i=>Xpga9`;^>rzAen+bA7MfUxCG5T^}n&f_>%~@X&V)-wO>< z`s{gMJ|rH7BC}msoDzV&VG^^Q@hMVeg878&6E;CS=#y}Y6S+?+icM48EnOk|&56GH eXs`V;SQH*EFVWg_kGLBCe@O=QE%kpIS2}hfvfo_* diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_1/1 b/tests/v2/fixture/test_format_compatibility/array_11/compressor_1/1 deleted file mode 100644 index dba38807a798cb7a57099233c92477607cf151d3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2758 zcmV;%3OV(70qxf5cTMLW2XI`PM5CG*Lo^~mBFV`a_CEW0_MA{5C~`}q(kn&Gl-xE- zVoZw~RudQ3oR~snIOimL-p_u{2~k7LV(y|<#C(+&#pQZ^|Au^DeP4gpZ~fNyeLdL! z3#zOhVx$>P;W`~6WkX%*oE%1Zlzjabdk-uPWa^^vLc2}hkNwf=rHleWwQ>3|avndP zBDfN5Q0ef!81;dJz=m|DdOpsPUehT`!1c-ncJ9_chGX!go~yVx^{jppGLGl)QQYBx#>R>< zT*rfe`c@r{t|r%e!vUXAfVOE(`Zc`3s;jBt25apN1D)xsu`R`7BHunndrt|0ZDogV zL7AZ)uWTEb2qba3_KG}BM{$fq`O}Dx>?`Df-c9yn}{kFj1iWE_~?%1ULM@utKfj!na@*jb0qJfUEX2Es+a`=nJ z&Xgm8fpP<$EZC)mzOK}RtpP2_n&|k@T?F?EV*;Zp>)98csz5(0m8>GZgrX5g6r1tH z>3+oq%b&w5yx15m90N{$zLYDhHQW355pBp%;dt0dxQPIjwB`!=xJ2Gm!Ey8b zW$q}m4NNi9dD46s7-86?PuURb59DDMTZXTr@0qiOG0n^(6U;f*UG@|eE)1kQSP8Zd z(H0!1USTE?MV6+n^&Y@7%&0B0j#-o0uF63+o$ta9(B1}Gk;&qUeb4-p^k01wQFLAZ z>NKN^FxOT{m3Ug%w&Hj+mX(~@%3&%LJQsjNL{mIJgXc^hZco_E6wc zDPI$+k<}{z=Adi%32Ib^+UMR=%nMXG+su7E~ZLf0{QQg^XRerrqSR)GA# zp&qyBY^x>ByAk_?{(^~gd?>6<>C1bm-C`r?Zw8fjrMJ?HQ8vnx)4(ve%wAlX6T3ll@HZz^MdTSfK!3dS2;|2(= zQ8n#F&(Pq$MZNde?PAjcmEPgpbn%pITK%{O^f~pRX6c1;dvduJqDI3vYGXq}^ATv9ND>u|L^A z3U|^%lqRbS`+=2jl_fPWkzzT>;4h;LG1nig6@VFdx|xZdOI~hx>{Rto-M3T*eF6`z zOmb?>alNJai5kNtv2*uyWUas`dOivhQqfssF&GGA)!)g7RJwbf@;)f#nviC^>_a2U zKaTw-Fk1+})%xRuDIl471u$C;75{D>e?~51M!RqfCxDMt5nW{Ru>{ z=_mSGWQ!=lc!{&k>1ZIFKyP<+uzCySN+TH2ugbX^6zh;1iTk1iP6M4%vJ+46 zv;2Fle#|pGvZjJ<1*)U%>OALmkgh)^N3c<~S&5jM3WAvSh|#XAjOp|*J#d7j@BscE zT&O$(C)`s|#jcAc3w1rYdY<=4av87GUT3SRN^Y9G7H+6`RhNazK{T*g?OZ`%_P+BF zgT?zgYl-YsYPl8e+JaP*(sBG7a+PvP4$>T~T=!VA@i8Eb3lF?NJpjQMBSM+sK4m3> z&+C4no|*9~1>>mBrFF~+yvmtjWHO83QMjI8!nXA})}I1d_T4ZquuM3eumBG?Qv+FO zjki6&*8R0>CE8@X)XVfzdAc-H9;av6gXQsfa@;&m1~#?BU^RMyT=Hwc3m?PJ*wtjd zPQm=DV0W=Omr^{FT)9NfWsR%sjN0jjx(S(>~1ZLY<{+?9srj z@}bInIEcLj6_+EvK-QcWxjn{L%K6HcY;V^!CWb{I746~6xUj_K7LUV;ZSQu%Nc_FZ zIZHS{)lKQ>E+g*49_78+jr5SD9iA*k)`M_6JkCgi^~x&eL-n}Y4c(~P;@j-rOboAi z4-aDJR7E6B<_9pnDShL!9UfPq`x5^|+(Ud4*k%es0G@Gw<~x8CeuHrV64uR{C(M6q zZ^^mQnN|c=VV$xU&SM+c0=!-y24;fAVt=!e{gW)h<24s?2KNbs`g;>~>NCxw)mX#S zLH+=Hh~FDns?Jr``&zMiYFX8vDN&vOVM59kWxc@A{%&Iz;7Gz&BfGz&BfGz&BfGz&BfGz&Bf M{GY)80OquB39D9bHUIzs diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_2/.zarray b/tests/v2/fixture/test_format_compatibility/array_11/compressor_2/.zarray deleted file mode 100644 index bb53d586e8..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_11/compressor_2/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": { - "id": "bz2", - "level": 1 - }, - "dtype": "T*-I2erIlZijXjn9(4Snr9L6$JR_5provHHA#tTt*fRI~d%MZ;y5^AMA(y`^0 zCrhdvR3?v?mt1MU)H6}4zPd!bRN1Z*12*%O}vW1B|l zf@eT^%9oaf;QlQ3Fz5>%KaDW;8Q8N~vGSkavxiWs0jiASz{*{SMoFGhRd89^BI|>C zCOY1Ztt1;bn>oJ>HtWyqtkJ6OU!EFVc@P_1v7iC@A%P%-{2=DsWrXNbpe~X5RlzWb z&Kn*7r0)n|3!Mrgi}(jq5#o)OLmI8_LdgSy^MuY30>#wOU4rWV)^+p)(?aBz@{*lN zz>)woG&Cp}QyLA8_Fp1Ivq3LF1Jl|3%LG*)TTqHIjbKMVbg&IxW!ex5DJlKW{AZvM z68?&4&Sk9;;7T~Xd}8{kfG#Afq!4orXgQaHH#}+k+DD7vP&*x&=WO%qeLIRJVgnl zx@j>Q-F^+C9g#on^(=RkuWm^%Mbv?qymM(N^Y$ci1DxanL;ll+bh4IPQ&09uq-uFhqKcpOw!BrqfODvMP+pRs2U zBDKl%zCXXU!1s3JqmVuzXiW>c^-RbHM1p21;`BLF0j%Y4H$_%LfLtSgsC)tetG$-( zHy{^D_w7A7dB#*`k))d2%ucpcBoh99W_i8L7%o)TOP}rKjEOGOTi!fwiu-J?;Sbyb zs6dizHQ~WB;M%x3Ih-@(w6GRkPto|-+iLO{h@L312m(d!)`|<>rtRPK@Vg(n;2H>yFUADPTZTgBfsmVX;9`!yc{1~eo~O24a7G=l(b;I+;-jWjBlgYFTVDm2vqo^!RG$eQ-a>e zHcF#yqU$nB%E>WR@w%fe%?`40Ados`dbaLn!4LEx{z{Az` z!);`a&LS89+2|b@zhsf*T0B053!bc9JH!o4&WJWQ@e(z`!$dW)q5XJ_NWrJLB-7Ba zTeULv<+duEue-mZ|Iodwf^A;Yo}_{cONwT0XQ9{I9uM%=`1L-#fsUtS%qVFeT%&5R#R_Z__$WOV7+$owABA8LyOm zhDg8fFXqwgQI#v4>7jK;vYBXK1cC{~aa}$7I3ga;)h2T+!WDIS!u(AeI!Aa4=OSQh z@``uMc*pW-sno0T>y+j2X1RWm_osSN^0&>8OWRksywg9($j<|KR2y{xiu>5Vq4^S5 zShjKV?h0gQqQl~U`@ceP9c`#Kt)w-`_i#j;++yE-vuoziz`uJx7>?_IuH)oTSfcnw zQ)_Xhmewptw7EBRXw~^5G$Ig|dC@KVV?l`7VSvV}e|&@ZgI$OU%?*WYWJ0b)r0nW4 zfg3{(qxR6mf(3c}k0$X5((k(l-xp@gbq3?7F1#?fJjpFM$`s`o&Ck#~uEo&v@*65~ zBa+EmBrM^-H~T&;nz&6_H%P4I&#YSUb{cck4tG9rBO3eZ8s=O-8gXLg#b-)8Kwi+2 z*`wm>Qajv7opJLgta4k#jODuQn%9IIkav$DM;?9)p#_P^3?gRD!Nx-J_RDPcA1SM9 z3N`n!FgvmJkdHXV(F!YNr9;;q?Le^A4y;{UivG?ossL;a>~0{a&Kp}vG!=%`ctbo= z=X>1TOjTi2H7mtKQYt+gJVo6Lr-#?a((+n5FR#58Lb@q?PcsuHCbq-Q1A zCYHu2D60v)oH)dT*VwiDgs+@GtH+S&UNT17+y#;|iK^JHOMdAXG054>=D6r%{A$FA zr94qbHM1qZo46(BE$oRGoc~P#co=m0}P2s(Q?{aix7+0w%+u_@yQgi+xoI>cI2{ZgH|h4 zT}He8Gq9-H@`OXGQee;%o^sisNufJ{wMBklI+K2*+Eh7*upOiaid?6AfR?^6vn9gj zStgdEQ>5Bt7buF#L@ddgD~=$DArRi;UP1Z}YgCS1$U!PLxGqcTcWA9+=g;+VfghK; zjYr{K0d?j&G{o%9CKM1}0DmQO;4I3AUkWjM>wlt^^$4CC6q4smOcPb2Ccj{s+wan#}-Gnr>+KBMg56nyT zPclC|zN&p-FVDoq-N05EDFFW-OHJ(scM9-A5H)quuU>x2a$m+7`EWhp`iiih#B6)e zU6^-^HJI$=!DS=hgPF=4NFeXapFF(`JN{G$yFosGcEvfT4T-uuk!jOSiPs2b2A88? z$qh(aF>7{fKMJk!IEui#=I)Too3S|u;ze1!F6og(4;qw>5O@E;*~8M}5D<2-bZeM3 zvV);**E;e#iE-$y9I?4~{x!MHzIS*lXokm~ai&+!8%EMqMI5B~ zDQ`c6?{aJ0^bNgKk2t9cMphd&yUgTVpdFJGT2$CpyAH{6z*mU{(-m6Y7klv+#`5zU zBnV&f?|{BTPZv{bj26x@<^{i%N-Rh%@9okJx_+}1yvSN{;ZP87XWA}ROD&$7R+FZP zekrG`y28WX_?}aTK*?A~!BcKPS^jU8w}@5-&%0gU2D(X+CWoLqRZ0U0R$wo1pjP;r z9fXs%toDV)z@9h1&dHDvn3<5^w^3}5$J694$)>!OOvC?6A~F%RGyFx+pAmzoJnS$5 ztzo^6ntJHFBb2N5Yj?y9roF7vqC`khSE;UpQ*m(C&w2MlZ<01%B;g#xSz+4C$aUASIk!)u z@)8s}kMYi8BhrL?h2On=#EjBhm08Ho*Bm~sD%-0lkz-yKf!77+P}cpkAi#a>!!M5lP|aam_uQ53)=Eh_YRP$1KQODASI6KF5o;51&xe?299dUF*d-9LQ z&X#>D;$Zw=n!Mwl0t%axDmxeFs-BNof(LHa4ivSvuX`{eS&!vE#OJ1)>8@iWztjln zAWR4=7As4^@`F06c#UC)cUt_)Mz_r59$4U-ksqm&DLbO)=g@#p-P*ThY!M%@>8yJi zX^jQ7?~CqQ&_KZ9 ze~LR-V9_~~1hodidfAwzsckZhI(@3X4 zvNloMuJN3EOpjR&X<@Z)743Bt#r_pxU+#v)uRtTtd{~?C!lgM9v-4xzQ*e)Crr6CU zQ>kG~ZjCo}E0e;oVpv#6!xmqJK8AW@><2=UON5zr%WG$JnG5A3QBa(lKkOmA3j@NSz!k=qC`( z(~*5u0FY8?d%a?m*u++rzw-uKOzYI=1fQI8;tTnE7;rk%w!Pc0tdj55<~DAwHp!xRMrMl~_>@AuUn=g0IhEHExa0G%6HMfIiFd~E`PKMl)zCm*OU2$UIb~-Wf7B6snh63ZN zm;X8>=8VUfaL>&qj#gAX={YRi%G%<6%O4b@SAn`)I;n0-j&Ng($L)bBS0NK+_swP4sCnG7?oW9^5Z)+O&dpH zEnvH7F~5+~=~J&}y#fJ&K=vR#O^jfsW`@c0(ubW-U?fYYXA-9S>ppy+@W=(mwyRyA zZf&?ihiJlqG3SyIcIw}fdhdd$HI7;i4imE`pV|u46RlXg0H{J4Dq)J9TBCept2qB> z7FClEl{%mBu}jOs2Qq7=yCk~=?p;fdJRyEo_~&f+A7{&yK;k!5F!i7B<%cH)(zS}X zuYL2y4tPI%XMGm3kDBHM?XivS>!kdd(>W{Qd_f)~x1WZs7ia9m3~7ypYv_{` za~A^=MKJ_2dFbyRCChAdoIqHT{9wTMJ)B;H+8r+bmYnN^0A7k1ok3>thG0KJfEmK&~~xdVb+L9t!_#4Qs*!v%&@yTG)bXvy-y_S z&WXPwrnbsg$Okp^xjVne;KMrs5u?FKB7Z%vIywGRna`$*sa<`Y)Yj%(gRy)&6qEC| zak*Eu{&A<4;tw>#)5Q1<9YmWqsm9eM7%*%XTq$FHl~Iuu@M(}U{g zhO=lP332Q&m_L@hB?Wh>U;!k{6(N zqU~y;?3G6FpW8?B=n znd4cGB~MoGbJ=M%(-3YtUZYjd4~<*U;GM3u1eB!kMQwQ5Na+G%@3K4;B*udtQ!CnJ z3N0kgj^7~&F`)Iad}N16x>B1vMfWoo$LNDXbg3NND1p(OR(ual`^6^jl|5DF(0C%| z4x~LxopKP!&BX}a?hELi9^2P;Ww@8M5B7@60sel9F>S%XLN7|u$S?V%Ib1cPk+X8H zZ-DO=y!R^0nKy~_hf+xo$exF6RdMKq^;f&ATJKLBt_@=?cDC}`_<=k!#g=O|I}2@l zidele-iwsWTn!7!j<9P1t{V5}>ZMYA)!)Ci2m1*W9x#)$dHA2NjvM1L8+X`@+@-)2 z8(@ZY1M^G$Rs@AKAS!G|D|RUbMniqX)L(`<{i8Fg)Z-b>a93L*9IZle5a={|M2#sD z4{JU}XfKi(z^$s@$q|gtfns!b3=nwL@CrcmAYfsfK%^3*ll9f9x+mz2py^tG&K@LNKv~+64hEcs1AxC#xbiX4I-qDt%_Ec zQPd1-RohUr)OJo=dhR(d_vJp^|I7FN{tw^VXYNKaFhIMi6I`z?=@kP2Xa4?Qe*IS+ z{o6kC-Mytp-Dsh|-vSL&qn(3m>iEhvqY`c+IZ{OgoD8{;Y-(gG15TJp|0l5mmF#8^ zo<+l8{gJN<^-EYP90W;3A2JAM?iH|CP2o(xJ&e6&UmO!jZsr|S!Hn-Ku+)?7@o@%h z7Cd(!IR9=pu?_-L@j^Ud*eq-3e09G1K zn8x4bgFqsu!S{qQY0B!D{us@u9DDq{JpspF3l{4~ZZUSqi3&Uva-+-xIJ?~5;fUam zJdbD0sNneMqa2$qvyK&3w##wLsM#qEFqmQHa1_L@!yqa`6Y()*o{0R#D)>aVsA zCT6yWp`f>wwiU0VA-h(F@wPm**TBg2M#NARY`X6MT2Q6o#*C8@sCTdr}^Rc zw`mn6XT9OQX#+weDzPg4Pz!O6sYP4**0iLWExw-9|G~XiLYfk>e#iUKLW6M0FNL$u zh8-o0c}}z`gZb4HK4F?yh=w8<^NrUTcD^%}b@F}Yxhr4+W!_}3ma7vr4igZ=KfR7# z4ILkry!Z)hI=S}j!>QJ(KU!fvz1X0(x8>3^CkoK-PVPGN(*b^o3c)5X^M#<*gr6Zn zF_Pc(fcCJwE7Oo38wRj>tFUr4cdv`My_M_~CR>}QJR2-$5*ZoR0i%XH#9d8X{&jQ! zXN0h-hv|*-S=y5!A>sFOGY^uqC!e@=TqMSXJ%Ul&^s{L=RFp8LUUXW7MP18V@eSDU z+)Uwh`fh%~R^MeSLrG3Wq^H@SCdDmnUe@F;>yBWk(_!kwkoM1Xi-qYvO2L%@4JyCW z(TdrPn07Q*%>&5jtT~oeGgS^fUBQxO#zYpu_mRtI)`8{|oVY3f(qUKY7pgY?6RS#+ z#cK7P-UF&z_7*>22#A*W85_dZw&XgM3`m2!+;D*UhoTfi_4q8yID<6Ik zPgk&baWC805#MrN!(G8Q6|?%)i;ouEZk}(ND+&`9p3V%koAOYEiv!7ofE9Xg-ACDO z7t77t;wmycx0l1t9|UhRyH7hUZGi_e^I+9!hCbiMU;>vvh#GZ(%UxQg!6LQt*Rk3n zvMLpt2w)Ys=zH%Frh9- znzPCf%D;C15NPtL6SdI}hFu(XV{p~Unv*t>?uD;-iCy|Ec3KtUXHx^ZT;jsJ_%8r!;y=V7Q33UV90e% z`>PVaHKRG2DR?dqqW}qbms5@I1j4Ml#Tu}?x`SaZ=d|k=EDM9>q8O zX&GzhnS*LZw3efzWKxkjL?e0ytkUzA1k(cUA*g91H|bKpv6|=rMP%_@%FWfiC7OnL z7bnW7?xgIbQe`X0cp3$JwdCvjMA{ER)y-EQGs(yZ5mH6SYa6OON|@10S&^8j(r)>T z%~~o_Dv`e0duV8u8obmPjpJOd>$YKN84mq2^^tzg zR%!WxDxf%)gc51}&orOLSE`P5`t<5N-k?oHaj8jbT_c+eq7x3J^jYf3{=-cvP+zAG zX;jo#>i+C;SO%`eH%2Q!9U?1PxeFHOBiJNMgkQV$_wH@WzwlB9#R?w#0?r z^OURns_`gtzJKbFD!fOWQrj`d&r=Hco^-XCQyB_Xj=8PmJU^+?vr#AyRmT?x*`)j? zzU;T~=99uhMeLB5N67sB4{mQ4S9PCp>jmnx^^QGB3P@lquJWLrobh|Zv5HZu-}|j| zeYPaK=M3m78tdA?q`gfuY-)Z0Ny#x)mrL4=_9qd~jox;ppKWNi{Fd(lyi7nc(_wjU z8Wq%;Mru_UuA`}taw5vl?zxE$CYWfs7AN6WYTZIp|9SP}){n0Ga{knt_0KZMMds+|<&tgebZh+v<~8w&tE;OWZll>BDg@)k z9V1i3P(!G)ADMYf%R+b4NPRD0mW2n`?eca6 zJfiVJm1+B~yStvio?K{|Rg%Ih_rhEK-FgbY84GIkFX>?xVr`qUviecG_k2U2$BMG$ z@hi9`_#B^9NYssttaQP@1+i+&a**1GTq!@<2k2zJ@Ts0COW=Ja|62=t0Xln!L4uoV zOAk7vPbu)mm!ECy?MhQ|J|t*v<=@aaT_nkGcXNugJcBl_@lQOJMz3cNnY)!>qTV^p zt>QXZ)V4guWxU}v^*V@m4<#d33@OeF60$N`jc}@OWbuA25=Mn<0}hV zCuT-h)NN>|Imj6eyyQCMKKU3MF%yRzd@}{9X9%fy>HTrC#a**^K`7$2>)*39;crbx z?kf3S2JWY;JbN<_qEe!*v_}2$WzevuR{B_KvPPY|-xc1JQ9IBN*qN>U<*=&UjuM3; z*{?7&>GH7G2;W>{Xi88)3(p$Sk#{}w_!YDnAR#4}R%T~%>N@YnnV0@66mMMh!2J{N z)aj!ip_A`_y4fDCe;qff)L*_9v37!yX{J5(tDS#GX;cFpgV$;5CJ07JOS=l?h}_{A z<(h20KQ{8T?&yyihHdB$8Km|V+*kk$c55kkHTZPx*x99Bcxn0)MMF?d2SpR*M5l0D z$oqAY7E|rP7kBgjQZ7259g>RMh!cJh`c|ElP~-OfQD63Oixj?pQ{ z!);QVhT>z2xx^&a^Y*3IWU$06ty^G$fyHKDn_`4&WN0e-wg#3Nt2}P6e$e#M{R(hu z%HuLJKt7*Hs%iQz=+yO|=XX7iq6%-Ehx7YmhIchKgEH1ZRN}I;Pqk>$T@`ep#R03| zD)U2CJhUhwM?CGI+E8}1tp>vssggJa9IVu8hj1WjQa;82Kzs$rW?W0aDv+NbaT1n> S1i##lR3#(|fQNj)kNgKvyQuO2 diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_3/.zarray b/tests/v2/fixture/test_format_compatibility/array_11/compressor_3/.zarray deleted file mode 100644 index 774e0578b6..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_11/compressor_3/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 0 - }, - "dtype": "HNtorFz1XcxP1ziRI(~fg3Z45pbRV_EvA+YD{x%Y^71xyrFvscWYUMfGv(Tf;4I{<$ zSEz`53f4z(tiEH(Tq~{}i?S_bVN8}5`yaJ{(zlONAMBD}yvcYONFTuvv0wZK&oKb3pLBpbW05eKm zY3`260G9&$Z3{h(g{ER6x)iSY)|ZbVAfDtO;7=x&-{t7@U2fBo0H?vhmP7LGS9Si^ zpd~ht77Ir6I?89-1HRm{-HR(lr@&D~eo)cub_kEM7_~JXQWx#GBa_xbZ&8W%^5GWT z1xz-%Xr<$Pg*T?@Vww; z+hdYueS;AM6p+tuc2UCUb%Z#XIPm4$wZ-Ab2Y6@3-5^kM) z`dTl~5dBrNm~SrafKFK*87-{}1mm<(DwILwfN< z`Hov)i+r^-`Rb_A`p}=EPN`^8EB~4XjDX0_#Ja2E8K}qTKV*ejE9Xj`9(B zBSTEI)d#X!up!Kf<~N05-q>b7kyfNa@Pm+ zR+j&4NZib4nO6Wiy@u-37-|FJCA!WL5pl$<+olq;Bx7X$WiC$C&TD3E(q|-52PfzPoW<%)~YUNzw@9@@kAMRr21WSNk zdF3#kIH_VtXaJqhG0Hl0*SR*USSKZYDve($%nWWrTPS^|7Fh#i2K?OVYqquGKeVGe zbmsiH704xgx<_P3TA#!dsBBw{8z$|Mob&7W2J#D?F~4UO(r{LrZ|8>r!<1QCBt${T zxp<20V2;HB{0G_uxSaTeAMrnIlXD;=sRtd4)RelME~7~7JjzO@n`~E1R7c!;y!@1H zSWYw7uTOCY9OYcJ8ZpO8FqVy|$4l94wWjr|mnbcuZUayFX)aRC7oefmJ*|$Ehmv94 z1ZTu$rl)V{k2u@bOLTCLZ8rMERGs1gY1^*bcltWF-6CoOsgX^kmxaDds~FYzNT_f4 z5?W1m6ciD(;5J~^V82kqrlD$R$IRG(1Sd=K1__p+7HL_2uEY8}VQ1!pBd)Xx zIgE#LTJi?KW^<2HO4uYVa^xEX-TQE3tJ_C1t{u|WN%GJuUfIv|NW3P}rGei1 zSB-DaK<}ITiT9;q`YTcyGY2N}CvnSS))&KvCPWW@szw_gIWq|RxP^v)UL7plj%(>2 zKBNvC-)R380u-^4O_9`t&=SGfPx&>(52*+K^|gnlV;+-MVNu6VuGDTjMbIU9m%Kor zNoWz74txn(3PMgj4(#aHtMB;MOdfP*Y0Pdo#UQb$8&a-67X)HTy%#nl3taU$Y1| zt}e>_q;I!MX@?5>U)rv2DTs%!aWHiw>@!$L+G&-Mit(Y~PWGN709(z5!FQZPTn{e? z)r?&ru4c$Ud1nX30qkJ3^7X_wk*lD1%&HlyY!kPTAI)Wd|Mdmc`LH|W zhyZGy(dUg$WfregtGMbf~jiL95a=Ih8Crgjr7U>-rf zlTO7b+B7Z+HOk+9j#dainC(*mi1lLkgr-1K0aD2Ude;?JucAMqCt^FTKj<$$P+oBJ zl4AfVBePHYW>&A5kcV~AYjU#l|LUJC=STG*16Eh4p)6vG6 zDqbCA5n58P<&k1D;obXNCs4zJRVk(OQeB0E+J#It+6=uGmQ|v^*CA{%+{-TP)3sDy zFF(q^InS(w_@uU4Qpun3M!}1GC`ty*CMB0G$UVZ>_89L|52{h5d1OoQnDhcIoWa6s z&~5fsIs)fHkJ(=49q9*N2>8oxMsVu0fpzf){0=Z#&u5OkI?i&lAihO zY!BU0k3$MYz0j|NaR;oHv`dC6$q;{V9Q**T!stIEpcYQ|cKwyEwGssDoh02rpQUWL z-nb>yROy*L!mol>0G_}Ovk{pn4d^wlQjS-++&f(>`oo`5Otac?PS>N&f=1x;l!k@D zkne_W9!r`4+v{~sOPh$N6W7QTu?P9K_zpkKk%YX#bFEB_UOW$82yFyD4_F~cv{C?; zQ-|{9r6%zmrau+fkC$sVA$y!mxIs$g)8coq)2HI{V28#-$GBNZcS~YvwV|WrNku_s z61q4uqMxTPCmG;y+V>xKq`hIqB6@;LSg9mC%w;JB!X2DN+ve3;EmI3n&RYBi_*V`I z!8>|Nbs>ac_pd1aP*)L#;#l|6W||FFXOe-mchpUr2bV!B1Y1j;zzxA%=N|C^o@|HN zhUzZ<%l^RMz7lO2r4hIcJk!gCa5~xJ*zuF9%^0D&0jBN z(~2|u?@?!_H#QqjEN&1wIl&TIw4D*;a|qXDh&1abPEH}7R(p$jc(Y+ zQZrzos2I>vzqh#v8}}{Q1=A^KlAn$b2mVs8^e#elrf~$pAK`WSH&Qks6Cweoqav4* z0s5`jB231SN-T6kab%bm-|NOk>w;b+M~0>#_=oq2b)RDXI zmE%UYO$xW8oX~+^ch2Ehw$>}uLedpBnmMWq>5PIqJ{;YFO9^NIn*`p_UKv_y(Xg<3 zVmKrt4mwL4B_-lp$Y#M4Z#eBqy)#u%5euyL?)gZtITpQ+;|;OnWH9;D`+)z$J(yMT z*xJZy%F7e5$|Jj#6Vs7U&BV>}8L4RAM<~cPFknOmJD8O35JfhA=kM+GR z1NrL>=r)(cSD0Rio|0mrHOMd9fp%FbVmGQ2!S-nxZ58L7;0E-D_JEYf9|Oy^Ywr?n zhWlB`N!2@q8nPqSFO|g?Ml%&dnHZ3UR!slvU&VPK4`dS#A{}cP%^*-PDgs{w*X;x* z*pqw)Cu*J4SsufGNx5XSuMTuzKQx*O+e3k>2^NtGw>t%=plsiaZ*pS6Za$N$;j<~1 z%eH7#Pj#YAgcX4oz>#2B9RQvsS&z)f6hf1LPyEu79MU~oHU_E4HXJPV$Yt zW6lic;X9deS{iMRbi{h12*Vs;L!!^HV`kM%<0Y7swi~fes)kpJuk5vRkjaNdLB}=U z+342>v;vRvUMCxJpJd(%aRST(>m^BW9kGjoO;D_zRf35lNb9ISlZqP}PWneYp3jCp z%bG%m1F3=V0kg->@jXGJDB3`>iiJHV{<9hkdO9{(m@*O6|T-lT8pPhu;f*_!1mz`MXvWC3;|A{4dFqvza=K)kJ=SM-y2j@ z0SU$V!Y;r={ws_in2BBYkXj7PWCaw)l^5Xq?d}-3L~IK#9Ap@1!;Q!wc!hP(aeVxG~6MFh&G^z1`Q$$ z;(S~w;s#w$OoJnu*9>E9GrS%WO4|XLWtupGYGovkz^VBZ{3OAEH<=h2=dY58KNyJ!O9Z4%2Q7=xXKqNxX z`H9*$EKR@FDUhE;nzM_$;Kr{j>zb40c33#~2b>r04J0KG ziFj%Ipyh0-7KkhluWSf{p@?a!lXZX}q`yEsXNJ?u@Y2pS?M~zRgLV-k=|jm1ifyZU zCo3kKEwDpKo$`5Igy&*0bR((p^+NZExv(+1pC4RGqX>YmP;O9*c@6xPE>1tOF7*O{ z;-EX&L;e+E5E=;Aof-bvzlx8{B3Aq$CQ_|@8>)=70RGE@gk}ZPiWj&B*iW0MPDMxV z8eR)B$hxLx(O^kMuzP%9>qpn})^!X>L~xl}&BOWaXib>IlgoT#JAj8P-6a1hil7su z^qP5Ws88sXVsn^iFpaM1g>-J}Kv)}Z1+U#a=wB!@+8m@}$QPYV-DGo_D>HYWny*dj zmebi$Tcwz%3tui88GO?{u@t}#)DwzpUQBsaewi;=t#> zq?&^?Qk>?ua11qyjsQPfxx{H|jZ%|QjJ1$Xy@FapbQHarTpd0$Z}t|TCOKgUmv~Jl z5m&B13X-Klc?@d8r=*=>Mykwl~v!4gq%p3qJ543}=1bSMlm{uD8q=cYr3|@<0_N4by>(vtRa%M%elVX3e7vm*le&$aX#Q`A z?t;x)Cx3@BfbKvsj2bG&^IJldCD2)b4wj9-Y`ji?uPBIch}4GSvw^R_AQLnYFBz z;I=akTvrC7AJ}d>0x+6fjoFDR_FlO>yqmu}FARSqsXPEd9>N>MGCOi&FJu_bHH&yN zbw+KWxf$oNwvglvt86)V30#hy$bY#526p$DO2q|5m86Bi3P7!sG8j+s9>w&nj7I_h D&HbRO diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_3/1 b/tests/v2/fixture/test_format_compatibility/array_11/compressor_3/1 deleted file mode 100644 index d2c7a046ddd66aa7654f7ae11dfd9b6686760159..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2768 zcmV;>3NQ5n0gwW47XSco7XSdz3IG5U0002E3IG5owJ-f(a29n{02;N7DoY>|TWt=r zbiWz)8fGdmUN5dwMkqVLtCC6#S3$b?C#!~ zgtjCE0uYtmXzS>MdD>+rMwkMY0*nG)wvO6IXy-`cSl>j_qSxULQYCnQb&O;j+hnn9 zsy={yM1(^ES=YHHaN9p?f82b2tiSMKzM8f~2kR1;1C&;fBkUwC*$adg@)laI`Wuh@ zF|1m%J%uTqY__tgcKZcoomq81CF_cKR6RNw$r){2Un3q6v~mB;$tO3QD2_s2V2k7L z!7Y+osHD_nSg|J*dXKvZ-7CM67MNr1DjZCGHRCWh395-*s=1%GHqc~_b>?Gvglwa8 zr1o6&-Ao>Kzr#x+f!pHspdrpOa|}#{m1tEz{_{PwUe+kMO9-ad@lbp;CuMT|(mapb zEC~ia@<|*C4CfDts-p+!Nv{dS6?b>c1k%5p)HCmJUv48A zg09P4!v#48;SfD2CSaZcJS*8;5bDe%#@F?T`GNKBF7FQV)6ji06pseQaw4PN6dKPZ zdvJ0rB-SKXNu85gn&k95!L)y&U67krH5E7hjNSlEMG7D4875;Yw1-#h1F;kH*1uLo zs93&+_5?4&ip4aL^V%Y6k9WZ-xRq7LZRY*xlk}~qfUpD^1hY*gTXi7T;5OVoD~!ol zZJyv-T@UYVAy%MjVu}UN%Lji#Kr$$r_$^4ojU?&~y5fkzOou&#%Tr4qawk(*+l@Q@1ioV9y?;dbG3b5h+M@@%&U>A6USiQkeP-T zxIu3avBt}V^A#59^+bh2S$U#z`px>q*X&vkszs7->|TdqF5xzCcm1^fvne=*rkKRR zaC2a8TgwxSH8Zp5CcGamBWB<>K?x3O!;GhP>8_*^{zhd(pT)&CaKjO4A7sPaRVT9af5Y!GypbQVg{J@CDtTc-zkS>zl_FjnJx$9LH~ z{uoD_fskbGmN&z0aXHs0-IId5&u#^aqu68FFG7m33f@5ue%M; z5Wh+@O{ zko&P&a6XsHX=ItGlC%c|jZ!@H2&yNtD2V$DWs@RH@{Au@r+z=cb*=($S(=Bobv_;g z+t9K{F~@1o0LjcnbPLEfx-Fg}mjnoB9(z|Po=E>!W2eyZ8M9KhT!Ti?@&R=WSs&kp zP;bm($hUdQ{k;*brJiu_;Xr%>{pdBp*D2ljHE9|Obmn1u2)<~A>MDNe7o?g|4H%Z4 zuK!XRfoEPW7GbwTs}`v+&$;0JXhfVOCfPB5yQTLT-)3}A?3 zap~FElBe)^l0@qePhx-XoIF?!&OdOOSk2ty^y$-dBj+O4TQ`O+bEZo!^K1G+@039lRn*X$R4bhs4uD# zd|shE)7M%%&BjR();-1Ge6d`+bR!Uqf`;`Leuw?_^VIFRrmo>y#J=_c;Zf}ZUe@|H zrFVN9&+rKflte;*UJ{5!|_IS06hOdIRZJ>yM)*vBN~R&$5I4~R18F%w9x0H(qCk#JNS zsor?}2QDJDnw_vNS|?u)63yHNMw7yaqTv2MS24^d{V04-%4L31?zL(BCiM~bkkPzn zcqIgl{l1Y@A#>Ib0yTh*s9a>W`#ob2$)L8BczlE-$kE|xz zfk9=GS*~Shk6{3CAciYHH6su{D zczY-L1znz6$>kFNYGH}KWv1;C(MP}+fJRCxyNyZ2+lYmw6-pHQNOqvsK%O35k*A)0 z-|EH-h|P#tXB;yJS*f?+L~0j&RXgN?1U^1KK0ZD^K0ZqU_^nsWC$VVc9LgBFJOqnN zxN;}UY1_~*bc1+JY{Vwgdp%9D2>;&+@ro|as7>az;(8nf*F>o!%9}WhR^elC8L^VO zXx>xDLog*3J_{YTLJTS8<6wWRp0HklggqiDhGMDL%o@~kUdud&MQN`0g)~k#)>m?4 zkaNr^y-g(}^&ah%$?cQwaXhG(6fV>XFr8spsClTf(ze+MIMm~~H`HA}v=ha90|P10 zqu+Cv@0TK7_d~;e4J!N7PB8gO3qg4oXQ|zIBW+}~-rR!=&<+)ksCKA>lgm^9j!NPm zvXNvQn?+3`9gwr`ySX^qk@LzWq)>9j7gG*ru)k*RpwihaTq2wY&!UxlG<8ok>v3^j zmw(aSg+vCk!Sf-dbGPO$I}G&_HuhJJ6A;DpWZT+Mk`ROskqBrWC)HrXJ23g(tt$7IAsL-}O8Bzfn1|jnH@nm*J zG7RUNl=)o*m%J`YIzgejF`x=-(#m3JgS^KMfS;HTM{WxCA%}b#@sKm`&7h8Kc%vAf zKWsK4P~BNS=`)iAb%NeN%km!Jp~vG%;96$;Zxzj&aC|Tr2)ISpfO)(-U^g0$&Qmw> zNQN8`ldvOJ|&M8A41w85w0RU0spbyl+iaSaG diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_4/.zarray b/tests/v2/fixture/test_format_compatibility/array_11/compressor_4/.zarray deleted file mode 100644 index f105f83461..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_11/compressor_4/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "HNtorFz1VIKm26+S?rqGUL=u_yA4!5;EL)bL>5X3B!K_%v?H6J{D?V)qDiuxqg zCZfb85J*7efCLmc#c|ZTv&O?L-XI3c2Cg|EQWp>#-8Dda4*#m$Y7(0e|;9w?@8V2r& zk%`MRENTz|hzS)(o<|yNIRplXLX;bb!W0UsH5x$VM$8!}NrH0Z!~o>x&iG9TjX?1u zdf=I{W-U~tHZdv=DA?k{NE91U(Ae;R!4il*Y^meR53hs>gW9BEKnSbK$czqFo;0a~ zrwXf0i$`U91;uBFh(bc(2*A-LN7z-VqV5Wt1w^RJ3V^&GxS);MnHPkl7}8X!i|hkB zHhr4Jd5NQ`fFf{+UTThnaZ|tuccxIJApw1JBn0CFusV_qmMV*kFqfoVF^>#s>n@~*EoQZTwO@#>w=wYMLLqrz|tS~5pE-~~Q zLMO#0RHOMvnB4_U2n2$nl7WHBkb?}HHWoGv3Ldzf-Cc;|7V-cK&j}crY*f`LBB_mJ zcK4egtcud5DxeCfnOVz1dkA8@!sw`YiF$J;L`qG`jHpE@7-+&p(2=#VK7&x?u{7p_ z5UfUq7DH`ySa5v=wf*$3N3KVs88n8@2?rh>Q2m-AM zLRfnoKo-VOmoFdQ63H-nkl(QpW-ZG07l(_z?}Kf zRU$PjK2eEb2LsxS5^r zhqo%erUVcywBwlrWY&w60_q2}9Y-I|GjjE)qcB)Jn(xThoRN*{r?ODr%R*B7ZnBvHi0&$`U^(GlKm>|&=7XbRHU+il;l5<%(ATLWatb9sKgn8NdgL0At!(?AgZ~Ofj3ij#ioHU6r7&?y)d8_ zMUUx8lVWJg%91;4eOz#o;3~%nDo6qF7GsASg0f=`PE8Jww^)*nZ7{8Uf^ zO(fEj3D~rR=`p9i@nvj}5N=~9EKC+8g65_=j0NyO0&-4__+WB?WKa=Ga%2#BRC2RR zMAYM_4JWQXJEANyG6u$`M+%#!t3(B~nUI#^f>3*O@W{BCQRd0eh_*|?F}4$R2576h zv+g1d7@?Bou8JBGZ8=8nF*F&6nE;lYB`!(KbU;E69x^`XlUG`w)f0Sx z+=nX+SeU`BD#AdqWPn)@sU}4@h~5p=N8FG)ysPp|q?#_!911;Z+CXogzA&wj!2vQ! z85;vl6U-poVIqM=hZRI|C=i*G=neg*0PE+47X3ogR!dyd<-&Mn4QmJP$BP@3AYJky zM+%611%gm%P6mLqz#KSbO4H8x!v*qC1#M~;nFWNV@I-K7Fk&}e9av&##K4GECMCP}_QB#wKMI#XjMKBG4r zoP`VW29z3r(f$m;0{Zl!t?G=MfrPlyjP!uxNVzND$_X9;L7}M>WXs2JQzR3Io@@m_ z9y(`df}pgZAm@abP>J?h1q%vQa0jE2pDt9DP;hGdI)PY$EfA{HmoXfKjhV3K!qnDs zNehH?42e37CKxlda7hXO3#y6-Th$yarxOh!CS|~3tB2#gG@p4e0|1W>63`M$^8|vi z^~sO#i7PObA?Tm{K*3lBq0DAu2tDI zg^mX-3^@>$IT<^9qs3BNoVq{{xIqA|h7++X9&UFReoz6x<3^bruP|I9GBGMN<7k0b zMQ%xsL|?$TV^mNiVF1WUaEixdb%X@iEc{S|({B`L#Q1=Y4Gj;Ws@AN4p;nkZwz%HI z%$N~N5IH5Z3CgjC$P#i!$dHwo1B)b1x}aIGb^tr-bQYCGM}h}+0G4>bB~U3zF&WVU zR)8@KD+4GPNWB$_vlN_?L485Us>T{kd8$HM!9<8j7Di|uvAuyvn9IYqMt}-N_+Wc7 z{swI%!h}##C`ehLG7*hL^kJl8&ITI};4wySZHXDLgzAtXu}eibQz#6EFnwKl9!)@u zF#-Vw0TJ9Vc{+=l#K!o-h1LNKjY@Kn8T`AZ3qmNeP`{M~%P&)QGYWKwK$9D-%M-6Bh(LQ~)|7oNnwe!=qCi z8aYOsfH451Gh@`W5-7R#QB+8{nCxbC1EDnxm)Z89elcs&vvEoy?51tx+zKPp3$ zj^JT)HU)g>96EZ9!pF0#-bLK+DFnxf~K$6JJ zB~ET)EKqYKiAswAL{_Zmz4l@eA3-}18Gt5;AU;wX=%ImVO$;+VC>^p)fRb@UEyb0r z#*+l(mss*bNeDQwxIvy!FoB+e^Kitdu;@(iyT=?%B*~|A14ThDn9$|++X}!@jWJV( zKECXx(l|Q78=+x~kV25O_>lt_8DSDzKmm2l7#0*+WK^R13z&k25JM4ibm^kN+7>cf zA8;i}>r;XoxUC{~Xp;*NZmT9PO584;(Q1WsbOnlygfYAEhzCSp4^N>xd|4b*UdQhh zg)yzsp!dX-f6-<#?C9>3=6TVr`CDO!f3?q~OW6~trax%D--L-^@JhGcn(rT9k*eF# z?fWb7ub=I*o87v(yCwSD9^|}k!|9s(_&)Br_^@1wUagOAN<4d;9%EVP^0rrv zXxx7}&3SMC6(+W++|KU)9)7p{%a;CsE#a8nyyYh`J&8l!kKfm>S?$a{nq+A|cP|%c z{PHqt&Sfmm|G%A;^RjiPf&OpKXl&Zic;P_IQSeq=P&bzN;Ec&zf(Oro+l#Z*KS7 z_xsbzx0C;ceWh7$mv3IzsSKy|97gYy(|b;NuwC$j`49I@UcJhx&@1)r-b>McMmU)) ziJWue2l8H>w!xj!DVJNP``7ZlP5NDryyw!txN^D{^`&9*(5Gxaw~sTo&;O(BSEJ{( zJ{DQB{Y&-mhkc9W!qn%3-(`)*xhJ(~xoFbOzn|po-+Lv(YjRG7y!EXKT)&qSq*dYm z@;&VMn!A=;b#qy_>Y8)x_pbfsU%Q0kFQ@C)gN3E<3Qw1DcT2A#|91CR%euXLJYaCo z??0;k2f_zkFQDhi_Vt2`bKumNe9f(TI{m6=gEi~AEGx%pSIahKf7>I2H1Y-Rof_lc zc67&V`d3;`VL1J%cfCvdVo^6q*Q{0S-uvOW7u${euiVz> zxs%<9`sT6|QJ2dHe@-W87SE(dQJT+fKi5CsaPmAiQl*de+at#4Tv`jy-W&eem!hk2 z4YsTLe%|rdD{{^SBE8JnpT2edRWk138YE-WLU#7hu(toq_oSOU7>M#Ah=N3Cp z{i|&9R^Hstz_&JT-Zi4LT0r6F@TP-5&gimpz0+CW{)uXPmMPr(oT8`aZ9c=Om+_3z zK9}@4uSxZX!E|ZLNWR}4(|6`v`SZ*`FSsOJRE4^oPBZZJR`Sy>ntp*I=(dUzex;bT+>m`+Naq*`=mw>?U`w6Y1Zw zbX}Tm?VU_%yZ1af|5{JZx^P~kCvJMT@^A-<>iYZf+_{~3u5?T%i|d_uwxj;;*`B_1 z%=TZ3f-YHUQ?eEuKLD~Re>Q=WweiyUt2`u50GrIYY+tG^uy*Z$X<=e+Tx za*<9sv;SpHxW8v--*^7Sog#a^=VsrRD$i}PzjUy(a9#9eIj^Y@8MnPo`QB?=el?l? z9jB+THd#~U9!C52CEAg78hc)MzQ$^4WYf2$^SVC&s;AJOZV#q1>TB&{lP6iIe)aL% zrM6(Z7H(n*^YpCe_L%g%^s%%DqDXfq=eGQOZf7;DBU60OL&lw-9sYIycFa{S%W1oH zraf3qy`8kraNS)M`QfOra0Y&_wZ%ra+PnDLGd(?&Kl-Y^O{Z_6@fWZbx?is9vc8V( zbQ2}8M4GiJcwsXAy)D?_@2xUVQ`DtP>vr5*mgdR)EZEFzLyY&2hSnb03=y^XC63t>s7sHOR%fozbT3U_{@;Kcut)4VjW0$)r zj>3QP{c6#B?mhTv7s$fxu*K?eBi*K64!+qvwFti-j75L9EWe(c@oMD-^!+^6?pNQE zZr51_9*OH**Yg`$)9zQVS6u(z<*sAfeFo_^{c!toKA%0*PA^}yXd>7Dwlk`J z_XI4%Sv&Ws{6+VpSyW}!9YnuBVXywwZPj|SoP%|qESog^OQzdcm$jU-bDkxZ8w`d$ zJGUE70_V2+rsdW?48roB1+AiB#q zRZTwp{?l1>NTJF4kSo&Yp6=c(Qs(Ts=eijG77VO~JO8=z^X~TEW8wNaEN`Ofdykjy z`q!;8N%M~!{(5H1AV+?m#&h!MQ}?$~uhgzZ-96u}>Cz$nk9iyH)xfwg_zQ#VuHhAM zZ#|1I_o~0K=j)!hq+#Ofg}>#$7q`6s+>5Ka?mrjP)8{L$M*rpY#pSr*HiqBf zKhn{uTjcXzeY}>d+42umc^p``E$03EUdYy2|6Jz^JNtJD|NQ%Z(B*P9H}1}3;qo2; Dn1;ah diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_4/1 b/tests/v2/fixture/test_format_compatibility/array_11/compressor_4/1 deleted file mode 100644 index 6740339fbb7ef3372f35062e1e7edf19e7d4ddef..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2778 zcmV<03MKUd0g(c57XSco7XSd-3IG5U0002O3IG5owJ-f(a2CZ?0E)GsDoY>|(`^p3 zbibMQx@IcyGDEMhC!Hi$c6YZK0?>beXL8#615Z}fV`z_!M>{F_mM>cD@jU*vySsa5 z3axZQAon@-D`@b6#6O!IlKTSm0yP3aWZWcvNF_l4;>#@c0GKTgw&Z|e%+s}F3=N2E z3(XdsQQq5`0Vno?O30XEW1^8p%1J1M3X39Sh^rA<8W0I;7>{SjoTxg=Qs)(b4pNW^ zBhv0UWo~$~#q59~cSU7|wFmGR=>)Js5?TTU&x8s;B6P%twzx`AD5$1T-Jv@M`Ctr* zG+J={@Zzn+5vy(&hgBnKGj z8Nst4zT9j%J2LgdoC0NcZF=z=8O5A5W{O&9Yon4x&e#ekz8V$1EVfMqdOFxR#Emd1 zRKAhc34MShaAd+AIBn8^_Vn;V$jVX>8UxDQFvc@D5j&_V&ZY{=IGyF-b+-ddoe+$= z8}Nmmoi$=Qw!FbpCWqDpP710Amdt?{ED=2P;2rsmD9aNA1}@Jd#O3Ry;q=7UmiFVs z6$>OAGu*&)WD0;EtWIEQL4l)%wJkVHl6mb$nkFX)Kq1hANDLeepuBYW@Q|p96mX&FR6D zG2_bG0XBY;H=qy##F5vc%M>$WDD)`G`?8>n6){&hG-2b$r8bzkb3AIxVHdga=ypiD z@Ck8-fiSe+uLm$uU1I8?BPp&l9%f>Wl90$5GB-ey2`lm}&_a_IM;5p`)1xtbcDy!1 ze5r#3Lj(~pIL zgBTDpbu9v`h7(1^Xv{70x>8_~f#yGvqCg2Cw2|ph!v^0N)}FN6T=Or3xa}b@R{|Z zf_lQ^gsInpMWp1xP@;jZv>ydROkgg+{0%=&_A*;=3 zXwD5?Xee$-aqLrup?_gDuxVgfAgJ2{#|Rl-4*`abc6ciTK**NLR6csc>55qp_vOJB+GGmAfb@|W!63zSvBZTPp)nPbSlG}905Wo# z_J+_Ha+d+wjra!}(AjxL!UPSlJq-1bz{TD&!vO}gRfES4x3@h_kZ_P>Lam!A+V zvx$ygFhcc)hz+3?k&g~)oaml;&#eG6tT{O-wIJd`)lf;*in_`+oeJeXdfv%O>+9{$ zXQb;{?xs@rt={qckx00nR`EpDxYE(8khN_$zLA}Uk#~yx{rK+kY{K*`qw5X3t1+qG zEirkI`nKt<(R)qnI)A3M{E78xtF`HoFwN zb`E=wqHs2MPnN0;rnE{#Z!|@cuRqv*Q?zp+?dKLa(?~jo1I}+T{j8iZEzuS@^mXct zoNrI7H~0PZrBBh54Ysxasp|0bCD6xoZIM}{Jgus`XV+$@%DL0P*ni*CAvJ#M;O@5T zT9;Jwk?!bhmp|o2A8h#<3swJr#q-zJba(zpBeA$!#M%z=p zZPTuW$K##)TYc-K&Dz~2BG)+hx3};9x4oMs(r#axx3%%J^;2n=buZ03SybhcZ6^lr z`N?^F+YcwHYWAt$Xqhx`Ue9wdrur>?oJ%B~&fm1_l5ZY8_Bf6+fBsIw zp2I(i?k};t2|+~XNox+;^e=e6$s^z5Oyamvf5mep00O+DCY zc~$W{J!LqI(f8B6UsZj}*%-M-ahR|7i>5`PY^p1f-z&&%aL(IU>eSBvfM^`NqtN5b z!e_rt)rR>n{dO)JtF?>XWqssZFS~iE*p!FvOMHb{kWVNUm0msD)g;?z_x{~Jr>?lo z>&cq59glA%bgty>E&128a@mJ*7P&>yIJW6tF`&-?Z0oNVBU_$MjA)nxbZ6>pGo*-K~G; z%K+-{w>XD>^`CNE#1S-LE7D22(HvQl&EuPceiIWNn?xYz&FX;BJQ;d)7o&s8Y@PnZ9WT3OH4 zbg0V3^(us&hF$G3TvC0#dPL_`pRRKqL~W0xg}0}ZPpKb#%itVvU57H=@5@2tkI#2* zSUhdwT&GR7w;cAK&%fC8VIU2^FiqEfNMwyZX@96T*@0ZiQDu8iYvE7b&Mw>M9lwdO zXSq+VoPF=q8IDaS+V;cyRA(%_W8uS_bAK1t;q_GQ*eN3&-EAtK$m7m_H4D;~XP@#j z%1$+DE|ihDTKCf%$wpD})&G^|!Y3ZLcM$E~{`IMHVk@le)~VYbqDk~6a_>v3Gzr77 z5uQD3?4Ntp&;G+{Z<^)WzSC7s<}GcM;iRwocg_Eh+ZoR_g=y|Y8~rL}(mlOPyF|{k zow7E|1PgoK=6b(GrrQM)X;X%cNELZ=p3ZFhkuy96b$`)+p3Y?(r?LFn1^cbe9|)`R zeG*eBZp|8fuU(JQD+(;5d;KjSm6UiYC#qj zSa#a=(C#;$iHEK0|H@!Kp;Ax<$o`Vb9{DV`z=Jth5(OW zY9&y_I*wVE!|E9VPPRU9cMQaSddx24ApEq_(f({$?Cs}om-spjBNLC5oWimoqj4Ps zTzgcdBRi3?hM@%Q^-U*4U*5rbBI$?e@w#PlvqR-vr)i7yTp;e${Y5b5_*dWJZ{>dB z{A+0C<19z=F&~+IiiooBFZA@C$5O`Rb#tZDBbG!j^$%PlP=I;l3eeQ={y^7R*)9{ zlyV>y19$3|k)2oc5nSbT^c8vAP(IVM)>(HuWo?Jc-z-7!(rX$Ohq&ytP#fv;VDe5O zZJ3myT%u8I`Kv(FxOMBT|HB@vct-NdDuU4&8%oPiX`MtgA{6zuL z`G%J>aKbA4>(M9t51)Q4c(mTsWVkrHcgwP?0@+<7k%7F>R;DCWg3BM>!f)MB*e7_I zepoyI{+RJE5b3^ERDLJMJzA)0uW+QwIHG#hf>`^$x{!9Ps`p ze!2L9mKPgF)T{QlnwB_NF3gJ~ChIhfHMK?U0xT8H)wcv|@wKsMwXKS&=yEFaVq0#B zMN%6cji1Li5hetQlgr}bJ1*_dIIMc$0`x~B{ z>@~`4?R-3S+vxTCL>-L>PFHH?zqz$`TJ9Z5>sUhY8N#NW~E5ytmMAvUNuSEuI=y>G~6${AH~}1>-e9^rwaH_N*~E# z!aFl%*QAp)p_hp?oW=FM&7vsmuHXB|24@*asu}^_L8D28zE*+qJ)hS&V*$DPOCc-yc>+JQ6fj z>`0lIvkVS6s-22DnB&}l3+t&+85xZ|gBY;2CHwG6dw3?j9(Og8p?W}3@sYEvwC6SP zI4p$s`kM0bMJ%wOa(_s3`zu*FxAB1<5pTzddv3P`Ku_z67KfftZuQYz!*R?^GcU;Y zkj=|K1cdI`z0L6S&=eOi3?K%!?uX%z%uc0pmXb!J?DNuDme?-te@*E2zdzNJspR~5 zH|uLt2V-(?!)STdNS%i54HR~M@lC-7J~Sm;s4_1k)K)ca#7csP-aGJyF9vWi`dxz9!ce`&W<1Vo-wz!TaRpcR-rZKO*w}j zKXysmNT$ zJP}^5oNaYTs}islH*PHv@`G+fJb)3?Z+zT}_C)_IkS>3pd0&XsfpL+xaO0{UD9xFa z3ot5(nznEgmpb|>iE{rS3)`eUkt}jBLBI6PuVIu*N5O?=EkmV?9B&GbzEARd;azg4 z(S=68EueMe%?(>0AazNOntQ<_o=1vo25oL8uGPj^hP7Ppdzu7IPVMo^{o|YLAXK@f zY`E3-V?m-$H#pS+&Fe<&StelG_;xk8#3Z4@w@wevxI zf;!@FvZ`_e!axk(l-hB1vhS;RnQOy*vnY}1r=R`jrl$4H8hfk?tTYf{7DS8;3ZQ?is6`Z0gcl&`5sU;B21H1 zL*f^hj!dq>>4fp7vl4y3#{?zUtwYSIvqf+QS!} zI%pw1*=@!w3zC((|oDKvI#W94*P zs`Z!cvjj`?da?4&v!&1;>>nuRh!E%4%|B9(?yS z%Oom!Xk2c`0yQ!!y@>yEl|5S2Tr4P>xlQckj zGlcc>1?>dr#^xnh967k~cV%qLny%6`U!M`MM-lym5Yau0KjBVjfRdI4+=8_(7D z4y$PWdKzy8gw4_GO>gMv&FMY!?ah}i=R)-Dhg8VxoFCmN_B6A;Fdg3S5x+?``$mdm ztd!zoDx&|}vyU8jqpqW%peOy;81_shRW}TiaeBn)$^+lNL9usFV6U+XW)jJaBFG@_ zRT)25o1$3A9V6X$HdkdU%D0}+sPWo&Pg(SoMA4CLBTxR`wd?OF^z)7w$kE$2w{Wmi zl3U|D9Sx5yH=;?r4C3cIiW7Dz={OQ3jL$TI&Rghyddb5?@g{4#Z8)_hcPGx3gsjtE zrK&S;5*NLGaU#3A((olj0|nG3r#vaRVlKIzcf1O{eMva%H7@Veh~p>Asow6=?E9?K z#$KD+BGvJ5!ql9RQ6j)XcCMvtDRwG>0wiu##>==HBOBp-_ud<^d#`WFxlRX~cl+byjXA<&3rFNQa;W$LJvW74fTX_suK zVfciQtt(-*7`YWJZ_d;aW+sABV=JF%?({iuR3pjMjC;dk;ie{U^NFQ|_f}OsA)b5a zbmuvLa6_ip)y*%tEsXh9(V`g}>Jm2tuD92L_ zCy%AAX>#q#j9rPoK}=Z8JW3 zi;?XEmBzlzaWE?(I|pCv9GRdW~3Q4)1Aj@YXQMdeunfY_YG!;V?(AeediN3Xa^ywVnM}(So)~dtahf_k3-km7S!_Ty7^tg zEB`Z z#xBL>RsL??v^}pUwEY^Kg28U^edu3|xI1~QG&FOtEPjK~;A1i}NSAjvG{%W%Wh_vX zOY%y-OmLUMzLG7E#~!ykC{Awdt$N#4FNK=@ub_?3v0C9D#@V2qn{r&9Y97tr@x^=< z))|@aYa^K4J>R7l(MZ<$|Mcn%6wxo?!l{JFB|RHGo-!A4(H&hhK`rGA9>g{``nmxA zz)c*0h>!smN^rl69v|`?OrIIUt$bV3y{In-ydsf{+mj1}fUO^*8#&a+jy_hcD5#t; z7?-sC3=u;y`=k^wV-XGyC|q^plKbjB7L%d zas3=0>k%HDG4>*>k633>fN$puej8Ntzv^;cm@x62_5qt@g#5u5+;xw+p_1_;%X zeavBZz>ZF|$wz~lKbT^nHT?dZCsy_~jnT?QTSTETYCif$Pq6Qi6JVtbVDZcAz#a;V z127mI7O4tEV*oq`Kmb5A05|}kc@F#r4uK&6IK=k>0C+BlO8{IV4*~4?AO-|t0bmt{ z0<2e%z&eu3ZX)!dB&;eTYXo}gN=EDxX+)ZGe7!2sUa{WW0a{0ND|1JTjmINXpAfW-o>hX?=@ zy)uOZ%Lf1i62yY70+6_9ET{;N2axAreM7!0(cm7y$~tiL4+oZy0;0hi1%lTBEZEb3 z1N=ll4-!-l0uW%J8|)HL2FAei0M5aNp&UR#01g2j$iE`~aUfB600aV{;OKwy!w_iT zp9P`-UodkBoS_iW|917pf~lZ3@CAY~(V$8!=*A=daf1_IkPiH7#|?wf(}9)@5(5Dr zItT*+ga`^DfDGNrPXTW|*4uQsv!A^@#);c-2Z|N+iR%xcuT0 z8T}lJ$JB22-Q!;yKCaVU>9GSpiDbmH0hs8xx1vv^VM{(INWG^QMOmC z48IgA?&z(aW|kE;ua!xbx^Gdk$*Ouakn^qj*2Lw)Be?;4T%`b7J1YD!%j()p`)1L!1gv&Nnd@W!6fYN2L*q@ILvx66>89B6syf{U@D0#3iQQv45X CZJbpA diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_5/1 b/tests/v2/fixture/test_format_compatibility/array_11/compressor_5/1 deleted file mode 100644 index ad82b6bf841c5301a8a264fe1b2d7186e68ad48e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2934 zcmY*bc{r3^AASs8yDVcJ%S3sNO0wilibB?eWS1?X?1XGHwk#KAO(wEL*|K|)Wri%F zWS4cQ?914P88grKc>nvJ>-_F>?sH$~+`sdj>s;r+=+a@{0swFt0c-#O2LRw9R7=-i zrrLY+7g+(C71am$t1*Z8?dLl_af}tPua6h?GrD*>{(TsAW0)Aol0s=vQ~1kA{(`#D zO=Z>~ey#^YxhS}q7(9-$JAdodwdB{HU32FsFi|Mm;Ham>z8NbQa)d&qKop;*0wx%4d_rQL0~~+>pkR!bY?W^EQI1 zHFI~BY=Pu~J&Q{IOeXJ{9903eqy&6{)&ZQ%`=tWQJ|`+MbG%T?War-Kwf&CoXH&e; zDr?nGCb<^BgNdvbZ4~GjQF^iOo7Hs50Ezs3?F>0Z+Dt&XVDVLB-`^!j{@7+kkPoA} z+Wr2sh-!G{TMhsyF3x-Xfb$$LiS5Upq=o3Lr7p{tf(r+d$S3~KdsL`EN8&@(2O~Vm zx2DT#ziUQt`CKwrX>(buuZ~v1zB0J8ZfWo=C#!DPjVVg)lF@pzx0@g9?__^-dzlB~ z7due~rHL3T%`t;29;}_s+Bex$_iU}2ZvY@GWl)w-nj*{7P*oe+=kZ)-qom*{AccFc zYfSEgOI~SG@YCt(BSa$N9zpttcmLr_iP8W3?NIYgnBqf$2R#idY}Iq($J5uZrY|~l$}gkF zDhux~yIG_XpVzz)s}v%|ihbPQVs6T*-hCG-jM0>s zAWVNxj%Jk;bdM?m@HX%QRFZ;t_OIn~PIr%>V1*s|p+6!0qS|)CnyabllbH@O|G3k& z-cUgsFQ!`snlP9-aXM5jSj}Kjf@CIZq0IpOpBH%7#6G22M_0AOOz>pVm8W`_dziyh zjs#9pX3#3Gj@#v*uob&{N_5$P|07rI;1LXgzar`~U_aB7<}GwaHathIw9zz;<9ExJ zFT7+6S3Ptuq3vE)p3^sXuX|A(jN{yAgO@*_th3M`Ik*(Cs}%Cw3X68R6 zvv8zfmJL}<%)sB89sue(!hEF=i79Oe`A*yKqc`k5yK`%!K4uD*O7bPLy;YJJ*s7oS z<#rBnsO=sT&|d7(!uaV`b3WKkNS4iZa%gEEEa}y-58{_@J?eARh#R_=tR%oEi6T<& z*BqpmX(^gjC^9|0sTBG>l+UC5k3c|4ZYS$?9VeNHsXYhg$14dd0%`0o)VVu2LoCF5 zL&o&L04u#A7W5MyIb?!$UGMJx$ftzoCNl%+V#Kq7A)^^Nynd^H0YGBm9uOaNnZd?5 z%Bc61>u77buT>pW_jg8vC^Kxd(7tKq_0^6yyqg7bS;!W9BZ4Nsg-o}x#qu9LBgx4g z5mY6CTJEFrA`P}O7I15PL6$PS!_3A~4U-aLRuIEWrPHN~BpLTc>5CuM#Invr*f=kc zR#3-BL0y^rH5#QR(FWGx0wSA2sq8%^`8P2F`EgfKPAZN)Hcu=S_~z(O>Pd~m@ypID z5B`0#MZK;pW%+gOEeX)Dik=CLXOGZ7pSPLTzCY|DMECP#J2e4MmyxKO#mVGXA$La! zaf$BPy%2%mImbPWjj-$aOa$kLbDZpEI(&M9Ifd!J`i+8==Lclp`!H)yh!TVt#c#y< zAKg~Hjh99SgPwS&{|J)NIWLo=L2KY zc~@fp&WDjc_$Zr(u9MiLJ;G+=LP4I$fe5{rqRSej2xqq{Y9*;?w(%nl)5ddc)Z4P z2ZUCjMB-Gcq6Gdn8_wzQ8`J9%fHevWP~|NU2wnTTp-=ZDWP4t4Nd^<>k?nBiaU z6_eqFsoY!3sI~pTT$f;qV@dpY`(|i!H9D96Vog3QR714-&EgpV0Q*`?S7$-GH(vcy z8+CTdTXTetdOk{saXCLmIa3u8CMvGhJ?Y#UNDH(RZ?>DfQ9-IE_Opd;E!pm_lhKxT zt)#}lMK87E?)vF5GF8WOir8e1#!+IL515FB%$DFg(o*x?<2>3fc-E0>N1R(>BZYvX zUrWp>9%x*3aL9pK2>Qm-I~1kFTZq){n20(Eo~O)Ub!uytHENWdwBxhE75BoaW7 z0^(HrK^l<)61Ipq5c;-gU_Iyog5e=lh*4ZYG)M`fR8l6W2OyC~T&JCOKoX7&f@Bg% zpnyao5d_zXR1k-zQK^(b>hUIlL<<9Phu{w|kh&K}0V&6AXb^Xd%MGMLW<<0&2(}!M zh*Vq~m4F7ROwiv3p*d&4RxpGJjzOdngnV!~C|fls|I@?>r+{mNfC%|-QNSDE=4sbc zC@7FZA%LV)PC15v29xNieI0FVwbh^R9l0q9WDaWO#S0Xit)To7O& ziJ1-ts6!gWAdQO)qYgDsxw@M3JC2X%WiA}ZI2&LcbED+1>^ch-ECyEPzA@hPF6r0nB<;N$yD(3W5u}pIgYBi8qwbLDXVd z@7$zqa+U@lOy(5EIDNT7H229D*}JFsC7W{_PE2z(d08J5s-K^!6w`+G=WirHpNesR zLEj)I-OH$m=Zc$mq`73wV~`)q8y!3uGbB4CnoFnb^B=vX8{%4M@u-G#iQM0&8~OK< ozli|JRtmm9u90tjImWXUUP?7t8Q0IB`j31*_Q#)BD=6SBvXTdvYE23<%tSx+48ZQ-vM*|)V zZfQ4L0T`Uf&GbZ4BaJNeef*G_gV*a_O~H4>kyYM<`nf0;xAYW?9}-ol0fPNBJ(W%l zzw{prLTMCzTsc|V4*cQYf~F`d#fNNhLY(sxNC$=>UCEI$$Wm0_tTt=>k6<(RN~Nb{ ziM{~mNvE32TcmwVUEwviiHt}}2OZ@TNaQN*Ys&(3jdKKdLV8ztPR(`C0)Lj?Aio>) z#il5dw8l$Ng;3vLU_R~<<0{s_1w_jOr?{!&Br+)V_)(uZ4XnXcum#!4ceEaHsUV#T z@^1^<3{x7XdrF3?zp_JM|Kvxc2#n`Tm8E)5z6jN`9|Q&XF5R3Ky^-)bI1N3aE_?-9 z&5x3r@B=^-4-p5_tK|mnHts4t*PIUb^6eZu0*kn6bD4c1n8dB-C(58iK*vBCn9sfA zKOEK3%m*33Z9Wp_Gi}LbQbT8I{oQ-SO{|R@CY=)Aus<1HdQByD)ZNy5+*Nz5@YKo& z^YBSuzL-j7(36Dg%v^Q2|FkpFcrNay&Lp1%zOt+Sa=n0lK#f2?e?N2p-M1zNFIXSz zze5_dByDj=?Ew5l`jWh*WzfyVD6J$^;rm!xMy+?Hk)vdHH`VkAr*OuhwV}b9MGl5X za_i;gAfNwMn=RY{6YM7WPyazaQ*!87m3dZ+2U!XS82C%*SA99{pcA=xUhp)?vmL|JnZ;}ZciEVSrlON-24_+c zOs@1fzs5(1j_k@l;msir`W5Au#EO~nmH;mLg&cG5Fn!=B^&FfTWEKd1 zUo(AV3m8oI;S}SV@weCmj3J%P$KFZ!YaC$~>bpq}IYqTDKE`ghTE_M8JXN;~D}}#= z5ssDIHa?19Kq@WP8m5G~TryE?0IPK$UK1MS`dO}5@=-_@mRMKGx4~a+jmb;}@fU>% zX}|kZ;bEXkSim;tN*w7%3}L+|?0M)Eli;bbj&OVl!~Uj~`r_@El_+ab@Te*9^FlG; zCi8$99`2>?0^MN-X@{~(e!vevx02bAHpA?P>`1&Yyn(6lPoOK^Hx6}zTWj~HtK)Lw z9r+cT;X91_P`!+D$2si{Uf(F8VXM0sKzMp|0B-NHc;(;5uI!koXOH&@;R|hi@6; zgP)d0%4?7)>EuIcs~$|wQRAdBpgp^jKomm?!_6eOI#qoo?bZ=?1Xq!z^h>oR48wEY z(J2YgQn}Kr{--^rq6Wa38VNtAkCi;I=F)GIPec-wg{J6}a`furJV;YptF>_{iT~{Ute5 zIVDXEkB852J=eqJ0=dF%!|hx3cjn+m{Lx@vvQT+xbO$@!S6#8TTXq8+1Rs?(rFecF z`58Qvi@-~AD^$!!;cZkmISn<@rc(~H70&E_LAk-LN*kR@aEUb5`qDfOS1M0n<0F%( ze&A|KA#4fO;#JmVupW%UA8I2Ff&N?mmX8Nt2^Ut50MGbi(ir`Tl?t}grcxH{l-3nh z;<+gZeC6rIN14^`R|gdQKJg0&T+8WrDHqR|dcfhH@3~B)d3d4lpIYOK6PkiMAV?Hx zAN)M5lN#q0V}I!sy25`?k1%(~$At^fG@~e38SsiDf zbs;fTgs1SyDRaJcG4WLa9jXw=$l1{@C5LX zIgIS$C)`q8;H}pCq@{5+j<4Be(#TQ>%Z2U!{-mYZS!V1yToRlo1e9|oOJAvV;uH9` z^n>$Xu$7Cp{*(76?&7k|rrZ=@xxbO}xD*T6O}N~3gWy0{_#qquKk#j$-h|7;x$5vh zbLS;=txGd|6>0&FIO=hm*U7qIorW184^EaVz)ET+{W;aiab1m2`jctl zToCCP*EyTnizCPyWua6`(#n(J2Jp9q2upQK%QLr_x%4IQ8ZNRPG55I`cN+h#;8dzT zS=t>ax$l}2Lx*iKTL%T7h<%KX$;Z_=vTP-4+G=|`fmF! zVG|sR8>81Kl^Ye}tvT-V=p*M-vtepEO0zzY3~AS`=yRC3g;1q7tQ#{+AelO*E5V4o@;e=+*W$DA)=JdEP{sp`E z=FmR?N?0vi(q=HaQaPuq7XrDcEd zqk1Nh2jr=FS(+zY^!C=b!Yr`c{S*8WB=vo_}uCgVHVj6HrV&T zD{!?r5b}zfTq;`6UI`55=i++coGn$Z^sLZ-)ouk(a6=>>bzsK${^6R){-D=-!WZF9 zzI5Rj^|^NkJYT+rZ>F@6$1}rWAFu<|4W0tCU?;~0uAlIN8>VK1f6LavJb2vw5G6@# zNd;R@Es_D6ZOknzlG_+5<%g2Ci!I0*A)fes4TLbc;J6MpvnNm+xeq@p^rx@^4s=F> z$#ApPu$HG=N-KqXDxJF9*INzKS4cx8k{;tOA&Fd5erad}(^S;BtkMy5p5vHhcUGzU z(Q{8d=^OF|eMNd@{xA8Ft-aZXnGYi6m%)g%&fy?^$UK#j0$Xq`eLHa!Hr(ym+9bw_ z0j<+lwPU-9cO4!NPjPv{f*!+^v))VcX-{KjvNy+y!YzCd-GYPVtRCxalaL0^2a@10 z>r!zpH05mfBT_)7DJ$_)r=S*4tw1mGOsqudo<*`$rW$+5f5Pd*Q^hHsMEn;?aBc!C zjJ%$oN~^3@uA%TBo0o6xYC`HkRq9P6A%a>cjMBQ{PQp@8peK%!6>EvE&-~P3OQ%{}eQVsIWvDNq`6AI>I4eoEmMWd-oAsOUNi>%5}i^@@zELE|)J= zhNS*&pH|d6d3t$!im+p(yR^-h3-oX-?1FQTOb+*xUI#nS-CWJ3IU&1k3+U_JXtb(j9s#E5?(APIUa{Uyix;DnyGj+G%m3)u z5LduGw;rimtgGC|Xn-{apSG_L_{k;9NInSTlurI?<~zY>k5fl=pP_EB#sfio;{MvB zknw86G)K`zSG0A(j@{j*TX2-Nf?U7{LNiL6$Ub_$GzetFHuxmEVRh|!q1@oUNnXne zAk(>0uHjnZ9l;f0&Sm!($wRro1|1STKm{vZ4Ec$6Q$Sa8fpEbSzIM_*v_z!L2N+`*mp6@dHne{d?z8ym%~=56o}-N)ZiXv+P5i8h=zcLd+4ce42t|Zfy6V6nZfL7AqMyA*nzgByw z|E5L&KsEN6;xzoIlf6c;EogLM7<~aI>eo}vEtBc}NL-0NL zk2nEbwQBfnk|-x~B5A@u!3Am@GZSwi`RoGs3vh(LB6i?itX*7a<~UPJo2TUHiQHP; z7mDbf5V0>?uTdX&ERg>pAMxwhR>ln}gKGt6hDNw>Q7lOzkA=&jM)Vv16INCH)FYIB z^fFs%k7lMB7;b^nl{~3f-AG1oA4*Nl-`oe8uHc4rT5MBx6SSc+!FiBL#tN7G)9kJ7 z>r>jv^UdEZ66y!+ShSBWYis-{-c}X$zsawi6{svhl&#*2AA2W+l z2*aDzQAE=PLkT_QDAr-<*n?#q9eN62vZ4Kxl#7~4R7GZgPp;=xjQkSQZy zc#4AsU3SX>SS)YWviZ$$y19>u<$IfdDiz>7H%vPX&$_=iHx*wudr5=#D2_F0A^1l2 zXbXe=(t7#}&CjWo{wn%i-waPaH<-MzO~(HV4lr2HDzt=8QTLHy&MCFbahE?NMdC!G z&>3m|%+C>j;p0G>93b!ERB<_&gdQUfO|zoS|A?xLpdK=BK}-MB_R1_3yGao!TYCYj z%2e);QbGQa&EZyYv)~lHhtSNfk;U%I;>DiX$~b2?u60=O48;{&=lA^2eV}x#yp#zT z8TKt@BMG95;`2GXlSbWzH_&ygvpo;SFmK!~JOli@a4Gkf+LsxC>QRN*1P~77Th=yf z!$_stfysf+d6qXx%LH6Mca~7W&xMB^Z8-X~fg?7lkAn}>B2(76cz zE@=tgst{@*(=0K@EcL7lwzNIq+MxkSx#$p+s|=>>WFMYqmZ9$De=6_7O1qZONV2D` z10Ma3{#YPZQ*}?UguBN*MSHbab|StEs<;A28nZyJht6qrxIE`^?G3-){{?!BYEHtI zu&Wnptq|iX7=-R=&6uU;d9xi8t20I=SBNJnNyabKaA|gkvHdJqwVWkPUC>4Rh3=E; zF#Zxg71og}G|#sb&zD@s^J<=G*12D>2!p<*QLw@nuxc0t}g?BYUJtVYuZPB`d&(TFx zRGtTc6d&GgG-M0H(eP^!?KwzIF}%ipdm)wQJ|X6$WU7(9>hWWcJ=nmw5RRbZ+$*d$ zNk>pJx6D7wyk;E06QpG6rE(9f^$Ti7(pS`MxII2yh_`=|@CQ4BuUoQ3Z{f_gjKHI? zVhst|K|AN!auH}zW>!(6<()78(?)=H?yUk5z6YKS`ZxPvkB# zt^`9W;ut+fpATE|FX1^&1n)S{f*e65tG)Nd--ymMENT!sVhiFD;_ZS$lTe2=if?Z0 zmL3{2;5O2pS}46XV#Aq$m0yO&u^!i8yvckZ8jiQ97jP=R7fuhBOR+{<>pAm`^TEw3 zPu2M9fGW>b>pn13zm(4Ml}S;;5Y34TgmT=CzuBWGybX?}3$=FT{iSJ+4(vwV!;Ny^ zLdEg~<|&@;>BCIn-&3-1mSdRwM7eh8vQ(i&aSg=uz*A}NG0YvEM08=ohP z4K2q5K?rp%J0YD>o4|(tsqSvBW`eIImhV@%d!Nd{@ml&LOrtY--s z5B@NRuyvJ4l zG-{LUF;;pUos3VB$`j0|T<@?6;z93(Gn~kMT2^S9 k;$G=0IcQGs)kFL3SHR-%FnOugj(f~i^Z!dSsBfwN14!jOLI3~& diff --git a/tests/v2/fixture/test_format_compatibility/array_11/compressor_6/1 b/tests/v2/fixture/test_format_compatibility/array_11/compressor_6/1 deleted file mode 100644 index 7f243db58c9385b5e7f6df326e903b3ddbc0674d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2973 zcmYk8cT`ko8phGp0E%m{qZmX`L1372@7(XZ9W7wyc zyF)*S{nqNOi~_;6ar$s_9zUKUxKeHKjuHDdCWlcUn>JX4v#j^U&e}!T!0!w!VI8oT zH_$2Wz7E)}`xEGt{@H&1pymY9ICrQNDt_wfr%ocjSDq!^K)E~istZw*FdFranqkxt z5tdGO6jkFo8>zg*e<~K#cE%eRo2wO=Eq65cuU;2LHO6#U$ zr*e(HWDm1BPWU;bNVUBI9|ZVaYq6g8!N_eoelx&OLa85mR#@c0QiJLv`uT$uj2()T}=%)L~Cyt z=xjfYZ6THr1@ht^mTtL#lY5jRpKK2Kwb%AP?UT+FwdHaifp6Fq>3mdTruEjXO^7EofLjYHegFF zBqk*HuDDFKpNL9GkR%YH7x4 zu-7_I{xy(8G|&;&EV!D(+XK}ZtI;U>ATrc%mRp0jtu4>QwXF1n7N zqDEzy-LCI|P>Up%B|qU$n$yKPE!+CJrXV$*IS)Re;|z-I6~PL>R<~2iQGwk{oI)a% z@A@-)g$=d!L3@obVVL80{uLEM7csZcJ>nVqR*Hh%(R}d6`a7GUM{Z4yJ{`HadoYTNR7<>J38gCO%HZu*s@}_JTRqxqqch^J5IOFv zt{t$^vw`oNQU-^)-h#>KkmIV}37pZtk{9@WsG59cq_Dfp6snt%;Eu(jde{hOa;e8? zagEK`%yi-Et!?xMBTR~p8z{6w)wCBqM??A*_c>6vn@tN;dPi{6#nZBB_2(Ya=ha7= zr5DNV$>mz88Vw&?trQ7Au^seBF{xHp|5^Dm?}J|dIO{`YzF1`DnUOr_bhBsmCDjAC z-QI<)(;bhJ_FQGg!hYo?{$&3s+*u1%nyjwupREF`EUAHs6w5&-e+6ZVdHxWs5X`{S z%`Egn@^T|$r>ckRzN9kglXysFl2c<&=q=0-)EF*_ox8UaYXwHp^HI2viq08}!5|o` z{!Bii(%tivwqOU>gf!z-Um8*VaqJg?*-8g`AmLOdqQ&H0Glf`czhIBjJBTKkFVTe4 zuqnsHSmRaYFf|@o?uxi{rjRC?EHfHr!Y(L6K5va}o^&p=7E(=2h#{Cq^?%4OjM;h^ zlTdw^n4&$_))Jw1NR!+O{T7{x`ikksGB^fZ_a^hJjH%+}ZKs$#_ee^zO4MWQQhAQu zfro1g*=WmxUdNi+0_mDMOPS5wR~Vsz*s2|4-hk~m22Jpc#x`*s-MFhzKA~OnKgL^x zq5KwYAkz-K1+^CIH`xb%?7b;}qK1$&^il4w>@IPYwXk24ySp^T3g#w~;Z|$v6koP~ z#l}JWA@d%8jL9{}=q_!vKY<7_{X~CjJktcr^g41x(qk?f{ow7TTk!({rJt7#W)^pw zn&!=~odD8ZoA5et)2Og2@%@@`v@d|92Tb5JGQty)=UwhZJnzxuGG3{@!B$h1+%$PD z+)(klE*q7DXkfG2xq`v${TCnxi}!cY64|NLa;t-D3sOx=$MJ8-Rmx>KSaYy)-4n^i z$AECIL*OOq2?)Lf5y}krX)6(YT=xU@+>BQ#7)N#4QOBIbtDKoe7PAN*gX{SvY#X0r z{b`VG-vjdl%Y-ut3-AatHIR+gc-!-9-JiQwqD{sty-eRBPnTxOh&&!oj+^Jn z#HMxxtVS=9Oa2}3!u#+eb~Ra`Q?Q^a#9d;}r4-L3S67oND>;&BIJTQetQ;ICEAS<3mT?n)y&+P9^1ec;`RD)FcT~m2bh)YuVgVE zuepe`xNjiL--oDEpKBhi#u}~;_6OL*{Jy|ab*{4B*OJXw%c_1&iSmRCt-vg;Ue7CC zitZVQl^Ws$;h;L6D#ASb6P`fZs&vmOiLl-vAKkO}L#Z28Xv;DlijQEpJyI)Wh8w>b z-|<)b^)Sx$JW0O<4NQ==kuiNz(j~SOZYQ7k&zGkeUigT)AdS{({=9Ysl^b)kTgD+e zH_+bfbii=d^EQ4m-sR8jJ3&8;dc}@+DaIQu!dTzTvLjJ1eS&g-`!u49xDM2Vfzl6w5 diff --git a/tests/v2/fixture/test_format_compatibility/array_12/.zgroup b/tests/v2/fixture/test_format_compatibility/array_12/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_12/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/test_format_compatibility/array_12/compressor_0/.zarray b/tests/v2/fixture/test_format_compatibility/array_12/compressor_0/.zarray deleted file mode 100644 index 3be17b750d..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_12/compressor_0/.zarray +++ /dev/null @@ -1,14 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": null, - "dtype": "VbEsYfRa!U3yp1$Kx0nHD>pgi+ zrFS}6a4(c(-dl$2nqGp5;W${>Y=ul$E!epCFx|WT9vOJB2?N#R@R2LUE{mrm(W@Mn zKK@59xA?MKTh0O_EQ3}4F;IKpFyyJ6N34y7NP~D9>hq7fiNw;)ygl`G0ne!D_swWy z&p}};NfNt3o!a1JbFTVYY4Ax1Z!B*%tN(hAM(knrIyl}`DNUrq+Vl3f?+PtRj8L_ z6y6|B;lpIs#m8vrAqn5M{f8z3D?wQ+7D^Ue!yWUBNP%4#wD!NHhmI};c}qiHMSK+a zR(~^JDiA`|-=BueTs9WEVdA<4}ycW#@32MOhucGjvSpnxhjK*0%KI6eeXZoo- z7v&V(nSxV&P`G9-=`Z{T7Cxu3=UYC)@nH0FILEeF>Tz^=&yeIP!-&2S<}Tbr9|-5- z(&?_y{@#c!Gl&G=qekQy{J@ZHQNaJ4kN4B*Gq$vPpv-P}d{$lw*%phaeQ`F3?VgE| zd3oe&)fjOyFocI;gG6?f}ARMsPF21~RvM!NDv=^!yWrCdL9hnaelnesL>Izm^0~OcUwm zfnXd_O~a!2zu-J_gL`#`HRetI%UHIKfk{Xc{oK0g`tU%Yf1GIt6#3vJKFvW8{dFpeYY4ug%>?`X5xAZ@lCfrZ;*39N?Qjo^nOd(E? z)k``(^AHu3-VPD8mi2}|!%q+b?FO8^>P8fW;2|9^bbfKQ3u(FTdDLW ze%^tNy(sZuTu}jEz zb$sh0MG}WVo(YfnPHp{EQDu-6)N1ZT7<<$kW*H{kPpXaleip0%+ zOxb|#r-C3`MGGwUZ-SEbx+K%7kRGrLXP18NCeGXVcwr}Z0$nwm<2&+=WIm6juM{d$PeN#*>kb&RGo;%+Ucs)ucHk4gLfbwUu!E$A>pv*Pt&TW?&I=+@ay%Q<&qji? z^?3lf7}i^N47z62a*w@rh8?s5S@5JWv2*Drf^uC6=-g?E ze3gI5zd8kUx1LJ1-OuBSEq_2~IGO3yPK3sQ334D}0W10x*@P=5I6MCxE%_IKC%(m? zZkQ0~LTe1k&ZviCt2G!kd54$U`r^Y`9rV8dSw=Nhk#>AuiJ$VXGY8U)$nJYapt6h$ zHXG__PtrwLEf&V;``<$26$8L{&*aQLqYQd)YM6`PB4}#FC%R(%0$dD;=3cRQ0Rt_P z%s^x4QP)O8~`BU=!@RTN1$gS|4RxnV>EfYAD4b`5O|1y;My5k&W(KC@ z{ejqoe!6z~1(>EGOKw#oIa9S4Zk%_57rNIdGg}mnHw}~f*S3hay38JLUYH8W;(^rdSR-BYdW34W%kZMkoj@NZ75>VK zAy|CHZ$g3CL~6;DS!LvFNeaDsNFV2)zK*;gF1dPkGYI{<1a(?5c)Ym`r2E|2BUd}k zk2bo&f2YNecUp}1(0eg-pQt2Dd}`^k9djYR(g6;fZ$Q<9wdN_eX7XOy|6!#%C=Gb^ zo4np~9%WnGXP663xmc%mKgSX zQPY}s!rWgE>r6f|OXNMs-Ut4)X?6()MHx~@M_p73U5Nc*HDH;36q0Utplx>qK2)!x zr})&dFeHyW(9OoJyN-i4EN0Uj%jm2O4{p}2=T!VVKTm&;g?X>>CA!A(7$%>Lg4Hey zz*KGy$G&D0XxFOa<2UY@Bm0LgEYiWp%{|P9AGa~#h6ZTgsbi9DMZxW80{*_}M?BXQ z!s6A=7`%EncC1(m_0lKt-y1LbXn+g1i*}+_`5j34P(nk4V{u!QB1o2WF+yXG=C$`N z(W<@M zSH`{U)^~TX_&4HZ!@Kp0`&VMcR#E(Gb{Kxxt^oBp?|}O~m+GsjWA~gF#58U?3N8}C z+OO-W)nwf(E?W$*ByC~g!&cZU?hfys#NpWmyFsxn5HeqD!h2O~x~FRz+-)u3^11oq z%4Sp4(^?GA``z(pgDpPn<^UhxR5IEq4L%z((c`i>T>LQ~mum;1^$vd2a4LiP^g5c- zFV3m^CCEGU@eU52y#o8MHpBU8+i{jcG$Uib6nx%z&_O_1mK5Q5YCX}(v+)qK(4%J&hSTq z-N!OI@25Xb>=x%FXtm?7_y4G>SuFW`$PcQ{Xh6vaY0mNXFYNQvJMrHS7y6?upKk1l zWF13Jw{Q#=Q5mFR)u9@g`nf|5WjZr0_D?VII>6;{>E*j11eWQ_xBaL zK7)^!62+m@&(0&78Y0xvR2taRGtjUm7yk_TWtzWAs**1mwL?fq1U;9JDer)?(cNc9AF8dOVUZ(3*07m3^r7VDT;BDDg}l6; z6ztAaAfrmR;htU`UWnL=mM5y1r86h!?No7$ZfU1~Ru{pU?ik>%;F$~DZG&sJ0`Rah z5$>&ZMps8I{3fG=hb~sI$?nIQ_+JI&ddvh|D=-GPWlkT*|LsB`j7x^VM^E_K*4`?5zcWkYbAO8dl@?Ss6u-XKXKi7AAg95Vl(W)6OsFH@wczk zUt9@}A6LP|<9o@4bH2>4Lspm(Sxy`GZilwLv&e+MI4pRR2U6udM6L51Hb+K~Xme%! zS2GA=2Op4{Q3d=YJQI??++>8`y~5~GXZV!&h(3K$1@X`SvUf~tffLn9G+j=??MJ_< zDSLxheCxvRvI109h@$km1msO~!!0wlA@Qvo&A7?Os5BW;iA@o(_ehAD=A*MjX<{aJ z%uR&*wHeT>AkR7{$>N5Nhq&2!2VSfYd16rv3Mpz7P$E z&Vuh4Rm@`0s5V$DS<}`$eL8P=a{dTCPBLGv$78>e7=4?h$$l}V$5Qz@i{d+J)(2s# zzsr_xuQeoD6`x7TjbP#@5KDG1G@;ksS{XI@ZFJ}FLHxIEJ|}#?DHPxE#s88#iR1cG zoDlg#TPs*Jx@^aG?!S*adhZgkLOtNCa=@bv>cnZKE4K0p!YUhm9Q$blq;C)<@&mBq zQ#6j7Z6MByl5qW6EU{TDfK`2lytSdqObTlQ$Db8H@lFAQOt z{>n0gPujsVSCx5bE5X_Py8!J&rm+62El_vsANt#Tj2bE|#g#tYa9pk$4ro0joSrf2 zJfw%ZgADAxnogFllc710Vqm1?1|?i2Y7k?Cc8XW9#Xf+{zahq1rW1qi^-;hXU5FMp z#^^y~4=}6+cXbm3Y_O zazhjfd>+NwEy*DMYjSoj*5oK#?0~-4WyGSTnM`}}kf=X0Kw|J3um2RsmYpfwZS5Mk zJL?iwM*#LEgh1nD{FMk@L?qVAfnk3fe$QEhJD6;Oi2*qHubbI-As0h5+py@6IWRuK z;CDt9&t)~@_$CS7mAfCHUuA$fS}+^4?CYr1(0^D`p$xbA`8c8&@kkFm-_<8>a6iv6r4|9bH zK zNw!dKZUU$Oh0tG%u2R0^7hxtn%;hXOM_e7RgQ=PtXW+3gUNzc9Jz`TCp3+?=wSq^k zj{l-jQhDff<~1Cym;-jJUr?*oUNSf9Jv7(bp!yOgA~#rp3fsq6!%ijOT#Lb)3ZPT( zt%ax0NE)R6bUWpK&73QfYUVNCmZR@OuU zoUYpf0SD-}{X;BLAL7x(Q|$TQCqU#!AK~{kC2Qh}$VGu1(h&F?Pqih2%+5Y`xB3#O zjFsl?y7&y%3MJvD0yR!vt_Y_yT>7hO0&!A-V0qq@oj{^Z4z}EX9u1MTQOqKNM{>lIF)@>0T3{1cl@h|ke zbtay%*-Qh&`=GTo8@lrZI4fPgk*;tlq88_mZ)Zl3gJ*>3+>KTsQ5cJ{&5q2rea#sA zs+36oT?hrykC5+736}O1pk4nG+Vr{-cN{5)BeuPyXzMKaxTgiXLJQ#GT1{A7riYiE z?qKTBMRwNX5h@thNexmLqj0$~JUyhzdo5A}aUuTD#X6D&k$0{nF*LX# z9t?j8aRw7l;#&VsG9Vd=$36LAaH$mv%3K8ZX|1fM3t?REJqDRVGkjvV2>5l>&`Kp8 z+;0rP-HW00m!~+7r>&10%4c&PB+28`@;-=NG8-43SH%}6tsvq4e2|;J6?j8AX2W|z zpjX`own`^LeC}NQzAgl8e1uS=SA}*@o|6ssM{kWD{F9dfId_fFVRi&vKEn)6_65^d zLHsCmGasxM$YZe2D=27-hnn|ebSup#eP5?Q!cj*!(m2WH=Kzt~bPQL_REIjfBXo0+ zAusu96UiSj!;Xsg^y}I?W<%X~all2A_-f_RGx07sb05QOb?U&SeHR$Vu`eXrCWF}h zX9Jcx$&jC3$d)gQ0=4C3Y|nHf%#drQHWlGy&R_&suxQ$JY#|1B*TT%A8vNO+gCWsw zuyS@Eyqf);g!cIp>yJs)ZM`^rJ2{1n&tHySi?itHj2$QbuKXdR&=1p1h3vMBbJ+0 zIH{-eXt=Q?QF`nI$(y9Gb~q3IDBZ`jhR4*&DVxq}RwMyiCNNp^Dr}T_Mn)&y#eXLr zLP|1&{`qrxD%=F%im zJ$qil-Is4L$2${u{uINKQ`MBl)k9OT4O5@Ik|xLI;Ul?7IC>@?eKhkif8+t(JV%RI zZZpPN{}zK~UmR6y2!)aL{9tJN7M^+bQ9ns1FsmNGFEdOyGMqJ#T$M_vmxj=+uYS}% z^aD7G?1ijfee`U67Ji$R4(cVFq59ckSR7Qs9(PZt?p5uuF{>X;dgsAetCLWB%bQeg zyM_lI`=Wex0yK%{L*A!QxT9M^u3d|SovL2!Vd=TBJk5^sEW|3Jz~7zCTNL|k*7vG$%Pyj+-p_^3lwkQDwmh2pifD_DOXH) zkA&jjcKmAW1HB9EVO!-Wvumd?&oH?Q(<+V=5mzs2XJ`r&HzL{6L1lX1dN%J+AX~5oMtf}G`;G0C9P7csFkw!{(hhP`N}SlduAu@i1$gPo z3Gi%J7=HUO4O}*@getibvNPukxxB6v*jKA@$Id&h@foQ;Ap)<8 z{P31o8vc5x1-4nAVB4WTbc((!{t`?lW7%q87E(tQF7k6;JlhNJm)xSKU$o%kz9Lv0 zp9Tj7F4CN{*U3E@MLNDx2%-#v(b78^Z%@9HVIqe?!R9yJdl&F{e*zqty$s)f-^EN> zk-#S2s>cgAEy*FL2Ij9vEmxckvuC>ySw-J~{a5En=8}mkM~=X3 znQYiP^AXs^wNl*=WAwhFH}*AVlBJnu5V>CvMU7`trNe@}oFxsQVRsMJ`jJTgJWA%} znlY|#qZsj^K-3w(Of@A=Le`gD*ci}GpVc=_zDHK5r|}71&N9YYsU|kdyb6l?gmF%g zHf7w_z?nI(pnru1>grcQxOyBld9Vv_M*0(j1xa|iWRyHlwBc+FiH7XwFWBg^P*`%( z3-tmlFkUo^&Aoe_8H*^utoIpo=5!glb>|(}5>%WPhv~R)7Fn5Z1R}zVu>QmcsxeQ5 z9F5+JensK1t;erNo>75gXv&GcvP8Gh9-%Ypz6{z^VmvqhJ;1_Wl9@l6MJRX}Mi*>I;z-=L3 zY@8OZ=Lr+LZBlR#>d}6vi~QUvg~mreO*%mi8kY%zmf{dTlbr`2<>JYcpMg|s{S%B6 zZNg@u2XyM+SlriFOk;k#!{O#g`k^hI${$Fl^J@Y@Bf0=8MLI}f&otsAYe zE3xWrgw+Wz=%=s6c-m?K`bf{fg|4lTan2mJ>d(We|E-7N@+tI>@e>$$Sxg;%-NsTg zpoC2H4-0)Wx^tV?-{MA^D5#@Kd=v$xtyhk{$}m$H+t^%93lyLy?7? zqA_orJr+O(6wT0ga3+n@H>1;RcVavDGYPleMw^XGpSxFM*B6bb6_ic1q1 z_f^_}cAMaa$}G^`avVxSS|M#+6fBIfA@@6{Qj4bdjDln|?H7*)?c-8#Y|brOb@~cu zcqdbNL%2oeB@Ng2nCAS2|tBv6Ly1TT5O=7OB zuZ3CPF46RR&*|e!i5OjQ51i(Ir~jp;gWK3F)JsYRoq+W){r!F#{L>5tb+xd|TL(Wn zhJ%;1I@!dH!Cd)3s-`1M4d06L%1mvj?mQE47!jvg%Y(>lsc{mw{~Beb%%CHt4q7E{ zp=Mk;J*FVbsVLGTA9#}NJmnBtp^^_>u7~mNtb?$jGMZL>16){mfS<u**CC(v62= zz~m=`_YO@bi;|~;@{1{?t9&KQp6I52Zgb(_xF*!TI04M|QS96x09iNpLRIloDon)D zFWLxiDTE;BkPciIlj3Z2l*GfmBeZe3pLvKxD;}S2h#edEvu8TouyJKNJi1tbHbr}( z??*ix^u9&fQJk(b@FS7&Z|LZH0iJcaFJ{bN#(nkpFm{Br(Vv^Yk!680MA0t{R5MdZ za_tS0>a-NExfWx%M;pCA-xSsE=+LpF;plYL5wa)y`ZLtg;r#_%i`%i#Ip+;z4G+Vc znU`>-I8p`E88|lEVHz;-x%x*pO)OCjdgKG+Zz zh-(c~@#3RF+`m>9(XEn^o^-*9HC;q&(iiOiGDfeTE5XH$<*;USI&=8cI9MC7jK*wD zVlnK4r7t^ZXJ`}l{Vu`3Tn6rx#nHUI9W*wI!R@>6P!rL3=qZyXbz-Sl&>svlA9lfr z%^#Aszzs52WI*n;XShFS1V#&@KxfS_Qg@P%HtLk%zN=U0nxwZRYg;rHwa2hFXH(JV z({1`z?JDk@%#ubbW#q4i3`X4?r`xsFFy2a!v)^wv9QwHw@&Z?(<1&7zT(qC+#98CX zkKf@~o)k>f2*LSVCvn-NN0zVmWS=~d!l^Ijz_bBfNYV^t1m8M9>gY-$p|umkMAy&@ zf5eFB!-EjAWiLrtErtA!C%|j0k>q#WfpA*M=r@I+#pM(TDw4+M%Tr0nLPDOtHDzqx z>eAz<+Hlk|07dg+;ZEc$(z*LT2yP3(V71@e(Edpd**OD6g2riqehST*GC*I?6NOLR z{n%mo6#Pq%BlQ>IY2MP}UN>!^4xFWQs#p%Om(c)&fp(C(s*himt%5GUC6Fhe%`;uQ z37hIK;ZpmznOS!J+%^oa`pb^`4!JBc-?94(4MAo-slb+tNAZ43ISsS-=iX?T)p zPyKM+y?eNi6ON&WZxN+74XSh|2FAaHuwTj?VXdkP2cHyX^=A197} z+t}Np?c~Jpb4#QDqIH|N;=#yzI#@NX*z_okf#m+^6Ct6(!K+&)K!mU0LjOoZ@K zArLFbLg&>1u$=E1y=>40)df}bma`Mxx!)d%^*<`R!2(|=r$Dz?F^-I{1F3l%!PlW2 zR)Z09-6I$s-z=g!L;(guuQKeddQ#a?1`Genz>39d$TXRcc&OMKlMEiwX;ObMcp@4z zkDR9!K^^QpKYlPVwx>J(o*@B4&*==gh0O5VcSO^l3wbDrM=nVdb^jf3!Lk-_*Zzak z?^8fh>J{G7phPHO1}^q)p`VXkCufg_!n@EBy5{mAy%a@By{8nE4F}+c+%AA}bsYH5 z2cEASB-4y%gLVB}RJq)RhW6LEQnAt8kDTM^@Ye&vXE?CC=h~uqlPstv&Vi&|%dojO z6KvzBbLNId;z~tx8WmManm^CvJnJySjPS!4YJCJV95cXc@ek!JyLxjm0OIbA|*Y+68gxD1J3B_2-Gpw~MC znWp+G`pZv=Bi@lkH%*9R=jFf5w5XGiS7m}F!E#jji9fs(41>YW9pvL(Mb6K2e<1Vs zbzGFtfuq;gz`EOIV63&54#x6f++REN2u#MZqw%0%a*O7kyFmDF45Q~{jpUt_gC^xt zJeVCq7I|GXtMYwE z^iwbgd!2gZ7nh)Fm$>NamP+}om%xEo4O+Fyl}exOH;=Z=X6n)nKt;=Fa_^%Gn)4Ts z?1X1{E^ix%HV*dIo$V?xT|r@4Yz!u7mqJEc9GGmJ%-XHZ^m=9`g8Mo4&H1-PQRoVehTS2U zIS*V{YeC$NA+iW28Aj7M>+gPp^b6EOL%OP2)ay=So9>12ZHaip>=L@`w$p-fLC$Db zD_vh+M}B5Xf@w`8I^6A{rJZx3e%&jw%cQ<-$e2Nj)mr>@^H}WMcfY!PnARTT)ka<3p z9&%BGBYyA9+aBq2R8*JqYHF9El9v|jqa2+1eg<5S^Vr- zr2JDLD~8iSOQanijQnGE+_`}F$CksBpv^exmUtXVYxwSdp3YCJCZPo%XX$!MH9fC!=uw0=hFE@IWT5@2aocU!PBJ$PwaX?gpz8wEAGyLTWNtf z@r66ZzhQ*160u`m)ceAF*|U=l@^Ygf`bC3^DCurx$u72pxQxO zHi@CK>|EXv$#b~6d^7T`n*8oDM%^o?LHh72r&`XLWQSGwhi8SV-4Ceh41S2&QpkKwkf&#M{h$h-m+`T$JOnn* zg!{uvczG-ez4V%3sL+p0nshRK+a&LOpN?yX5Uf)lQYUYF_;|Gre2WU1Yo@6vlsuRB zc8e1GSt@{(U6n!AQ8A*uxr;~~ISb+v9@L=LgyrvFO2aNpWh7h#(KUSvHjEF^6;o75 z!mtxw9ePJ!`wzlF-#uU{cpR5(+YYxgRG^{l8C%0FLEd_MbY8lL+{tr7o_7?iq~&nO zVG)~fD;^@NpVG$OrTFA|6v&#~hg;Ua;kI2dvApgKCR4M?aPD>RZ0{yNXRQafP4W1- z>Hs~ydIMN(Hi2`>d!c#O59a-QP5kKX&Cb5eVFn{k5Rtnluy}4b-JV*-#s*C1O`W?5 zTEA+M*X>~>{)Zq2%E*EI&yASFTZqg{6&UQUCi}wkarIy`3@%)O7w?MD`jlSqX%pl% z?0t!9EBlC3jxsO(o&YbwGX-8cKB4td;+)M#Lr{E1KE1nF4D=3+lEpzm)R$iq;r#*f zfGGzrD+@R}tcs5Z&S2J70~8R;$KWwJvU=kNs8UQ|?NqyM*gz4>R&t(94k=;2xccq{0cC{NLe+ P|0d_o&nqFzXD9v-)zKK& diff --git a/tests/v2/fixture/test_format_compatibility/array_12/compressor_0/1 b/tests/v2/fixture/test_format_compatibility/array_12/compressor_0/1 deleted file mode 100644 index 70aee3ffd9f43641ad17facaa984800e405e1353..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12000 zcmeI#cR$ws8^>{?ff8kvP${#*CENFNoLPlX6b(^n>JmytBqJ-?tL#<6Rj7ID2)9t4$Yy|lsK0`&spAa+QL)HZI$x5d(Qe!wGodc+>Tw+DlS!Wt+lafEYn4xkzA zj-2TEXjNGYM|Ikno%&oXQgR`Ca+M)jc^VkVc%e+t7YLZk!&13-%(m4X@cgJW1k7;+ z$%o-Mx4eS<*DuLhosUO{Bio?s{(5}isf|MZ?d0?2IrRDF<-~d4db<40HE4eLFCO?k zK;@mk6OE4n*sX91xie>xH^MH^?{gC5yXu&CV#}~rF@|^#ib49EaI*Sm76?5G#;G3v z;NF#Ipf{R}8j`|jXkmmMdi6L|090-kfQQ6F^6bi55J)@=5#D#`z;0EJRa-gb=53}q zN;-HxTLjv<$rKXvSZ76h+U&ZYawQj&YoQ7kT=6m8gpI=1ywFv&p z-@@+p5(YWvDV$6o@D!rT&(GArkl*Qpg6aiZu6_e zu>dj7s!=tnJd});t{S9iT^XxlcL$g0BO~4F4+234sdbwGZ?jk#x#%<><||0yPDTQ2 zg2Xs=K`Qv*TpZ5gHW_!Q%!Oo|8PM`gj(2sbDz+vh!E@6h;=Upq&8O!;l)y1id!0sa zE!hElKPF&sBpk16iGr8*IDB2E48{tp@W1mu#Boy@6bOc(Ys`0Uyn`RrDqMuBTaJ;J zW)InOSC)X4NFvDg=VPtSX{IrKGdpCxj(GT;;MUIcM`N8@sOJvBDBn!bmYWARw5E}3 za=sAgyntsZf1<7|Z4NrEmn8bHq9J}D!`LUWhPq{6#cdDXu)F3jgP8SBSfk6wQ;rem zC}frr&G$R0{LCxx{f!uz8dL(eBXb}~u>jpq^%2$9*N|9r5@&`6p!CZ1==aqP_%jlS z+r3c?jK56Ae+AL^L*4AKg(LRZULgmLgu_RtC0MiNIW}o!&?%X#z$)Q6Eh|Ipax;ZR zYL!qlT3{S|x(Ul_^XZi3vc%QW4!fKr!PRabF8uP?*sN(kExsrR=J)q=6?80Va-1Q!`h#XaqRf6Z_tCY=q(ylDfNcc_gV@*0QnwE~<6 zE4N^aQ#4iS$buB-c5<8wk?KRCG^HgKxSONUMCUtE&nPF27qpn7i`h;P~!fQ6Y0yq8wN^6_0La#0?&<^;1k;@gOK{X>XKr||pD1_*Wf!F;gtz%Apg z@FC>{H5ZE~>gfw%$U1;gaY};4s-K9v^a(gF+)XYS%d)wHY0R6?f$VX<sBKgyG2y zNNDHb*6$B#v_G1etAP>}2c@0*HHq@f;! zQl?+;5n?SGg9 zGv30T$k|ZiEel2mqtM^{1u0#ZhMneJ&>9|$${(ze#&Gc%uZ(%)qRn(w+<-;5ETG)e z3fod$AzVEN$GY{fFa0Z1^fn%;MI@MZigV62rqXXum2k;bLHeys6eIb=NsL)Bn$ImD zJ=ed1tRf$eNln4nj(7CLbzzVka)S;hJ1C2KfrEL=P~CqIj)#Y_^1Kw|R+BO`pY@iu zXG(xo+d}*yWJAlfUxMZJabO*Ud5cUxF(u`OoCE43$a64`J)=RPaJC^9*r3-%@hMVdGWU=~5yj?Mjv{jr|2K)2fk@q zi6T!AV@UW4-1PY!ji?W$XNA+KuVNC8e)PlHD-nRw7w4yqknU?fwA-j}r? z!cE6;k)Qxp6xTykCV;nfD~Xx?1Mj%&p=DYJBy4IS^$UbR;KB;>wtoS>Hz^^rJ8t3W zlTA2kXpGnHnvq$3_2@4>g?MHJVz`_$_HMcWmu|=~4X%69Vm%O_1zc?6-6HAS7!cj7 z3ggnk9BsKDs3ACoJR2>=k?mS^%%#Eb&qs02ix)oZwA@@)v^WSdw{BvmyY+&%mpa^& z*#M3+KckQAI8@A1!8HdIxarc$SUNuxi#pHYHm@g4^3pL>%k!n9yCPup?jl&WubuSk z?MC&fjTo{y48%IR>5Qc%tdXM?{kmuw)PLy%35`0~Z7RuY`H@AXcm9W+#$M175n^;_ znJknw+^36#!|?oS7rbe38*kA(EU4t-$AnXOr#TuE@7^aK<(1^6-hBdlQX$(jmp%4w zH5Nq7;_QF17ixaW63N zD?4)r35m2ttFmnNN$qS9U;dfOmYTv0iTh+&UlKjm#e?W&2A{}og*a&gba|Rb4bLdi zTU#!X&qwC-1cQ4?Zs%h*YHcE!TfhhF^6ry#Uq0SH%@NqPr-Z1USWC1PT4Db)V^FmU zrQhjJ`1Lj&GET&ja(z({zAcL#dw0RaR#)QkK9w?4GVxCI4{9wCi{nEkFjFrRvg6%h zWWECDNNpb-t&xF3-87stW=ZeOV^QOXIPeQpQHwwUp3623ymP7>Iuu)o$vZRn63a(Y zdJU)ooHVYs_b0NhXK=dWfZmWjWG&_^Gkd zE>d-6KF)t!j{@e?(ZQ&Wx@sDcqrq{|Yimm66gQF=jkifmUMHqEJw?YXM4lW{=j6}W zm!gS{i_5UJ@+a9N(Mbdc&(S`uVQhPLh?cFoNa~~QAl|bRhSlO=MV$&!7D=Rongln5 zUxMqYYGl)aMNlXsjSE_{QT*ov>OL=nj=P!>-+E&_7xTba5>p{Vu@yJ^Pv^-V&jGnj zb=Y|JCUv+Y3ro@tu_Mx{>=mq zHCZ$x+>AZ`J(Fv%Hw{aL*TX%(Z*=+9AauVX$_cU0VR!6P;`wLf!n}WS;nSR6^5V)O z&^Hj~EI)n}CB0@4eT6Z2nk~Zd@v5WiF7R=5{A@te4Pk7KIUIJ&BTJk^VJ`5k4`L3UP9hm$ zd6-wuVV^10f=BEtlJ0UEAFcJn^DPXv^qhq6yS78bv5-Vx%CU6P-xzCqkupbZjw~>$Fb;9|rVr zM}=Ims=9=08D0xpET+N@!4zUPa1pEZ#36g08Z6urMc(|ol@v$M{u>KQKptO#KZeER z$W31yUbY)w^%}syrK9Nl)`tA*EvM5V3Ug*}gr32b&=+4w2Ml~SvN=Ms#hZk z`Bkv%DnA{ui>BSte)w3*6CD(C5u-X_26+QD|;(`V&uDXVMx>>l~;7Iih zP61IW$6LC`XyATtXu9#k3%Uqwy}t7*rf;!3ez~LmwM^c=olLNx6{J3 ze28y|6jXePhRtVI!IPqKDrv9|)&$fsZS@4c?X`jLMUSB8OeJ+J7Qytd?@6_D8aO@m zV1z1jprl9>4vN^)8@ghUo!3E=9el{@ZBckFC7R9aI|;0t83cTmhbYY?Qg&I946MjU zkr@s&f5MRX^vB_8gHB945f8JC*D=k4P9%;I1#3@#r3=9X$^Q!-RqJ59|ki(iULEJ6@nT16drQwXD`P%TzijP=ctif0B8=26QyYOp& z1ZfXeArBVsAy*BalAW6Q7;wUaj*IJ4PShwZIj;e`Z5Cs_O$~Lj$;1AN8zg3-H(Y*R zLlrA*q19iC*4{8sqEt(fCdGV8O94 zym#d)YQ}#cx?$a1$&=}D{=zjndn_HUn}-=o`Aml$?ad%$s!nsO!{C909R8fxN78z0 zP)?Q~uX;pa&ZfUqxQIWzBZ!yAe7xj46P;5ZLHvz0TDn3CY6mtk zuE#E+@X~$^%=}5L7q-Dni#)RZwgNm$-;1+927|bL5QrK6gP(Fl;P<>Pw6-^Zk*GPC zFeeM2C_N-I>$~vTx-t?z7?1bt|HBJ^@jib!IAteb^7aGZog4#@ zT1dY6Tj8>yoA~I-PkbXi0NehgQQRGaQVxlz@LrvWsx{CZr6YLSejAL2l#-Ppec&@S z5k%Hl@bC}b7plY=dmzmDV_yh2@^m@%&E6>RQ5i%nGZAelD5=?%-1gO;Js*uv+9(4!{=ku_%UcmvNc zMCk9Hcgn=5A4&AOWdg1>o`%6ev$3OoI%R5&dVm)alRR+^A_nRorPBcZKE<@}#X#uq@aVQhXh4hWZ)W=yF!%y8N z)~hx%9*h-TuDXLLZj)gy-f^Z|BRwHf_d5D1yQ6QtCHlHYpw+K3X1Nw5Yogup0OSoprPl*mEo7AZUfw!(TR9M+U{Dzf`N#$jtw)i~RADxa&OC+o- zeTMDpb)o8NBr%Llpb}Q~q{gf+jwn=wb~$ z<`KQ5I?xozf*{o%^2(IBf)nU{i0k|Y>(Ml5|f;U0UIXhutyQe z-Z~9meKEo}25Z1}K_LpSipIfw9nzh(2?{(i$yJlx^uyLns_GaFVLe-+ZpB(gcu6cx z@)KjSR1NX`y?HdI@E5I(c*xpXexy~4rh#DSdXRI!NjE>Y!xzhR!Ci|X&u-UIsX65! zbwLbI%S_-CV;!8X+zetR`Sj_q05%N1G(P1X48He@+1Q=ZFr|4F?b$sKnp5>j@|{MS zzvL7dd@GE{_myL9Up`k~_&hH8A_M(#^Vowcs^GaqF`X+`$gTX_p@a4@=+@ba>-(bM znpGd=wHL$UICY+l@?A)^i)N4WT_z!-k?1!z4K8^j4SaGIA0?+F-{egSObSd2ObSd2 sObSd2ObSd2ObSd2ObSd2ObSd2ObSd2ObSd2ObSd2ObSd2{2zh;0jUEfv;Y7A diff --git a/tests/v2/fixture/test_format_compatibility/array_12/compressor_1/.zarray b/tests/v2/fixture/test_format_compatibility/array_12/compressor_1/.zarray deleted file mode 100644 index 5202ce93db..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_12/compressor_1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "igXDEh9-*sAy188Bu-+;n$zfpZE2gbMN`Q-zUPY1^U;|pg%>$sI_b-Uiy+iwJj3i zKji?rwMWDXVFRRB_lE@69^F1)XM3di^Z;iWwF;$*Mr`IZ})v z2ZHdz@m4Hwcfi8MIj9%@07B|C@bqhaxW72jJg+s-tb3n48mg=W4{>Eus(P34crTBs z)4J%JMme~i=t(4dCzuh@<#?`R9OhcFO+27TZZeJUV@3?I9S+hg-lm1*tqvF-MjrB8F;V>1J&d3kt@Y6 zi>D;fs~nd;{zosj__AAD&H^JWgH`@9POU^b5{B2@ybND(R+3|5Ip7!`9|uTQ>tBzN>UdU6e*_9JM)NU!XpS@@B}U$ z+QV*)2qYil=ir+Ter)`34lHC_VNswVMmL`%$A(YR3eD}1@Zm0%PWg^sP6oh|T^H$3 z)3wx9sF!3E-XKlk!(`UQ$7tyx3E#H;hb96mL0KynN)}wh9rKGwfn6B1_P?cvjxGdw zOG92od=&Ure=}by5JJ`8pN8#Qx&g0^lg;m2VdBYM%s#CHLJLJ`-8~ze+WQ@)v*v(+ zvn2P0=HvUV*8Q;2&u1x%sJ<%Ti=j*z4{@%7R>?)YQXldqVS+u0p~u9##ui; zV7^xMnTsFZ>48{ZJ-iR$Thy>rGM&ua$z>sZG!2g_&_tWV!wzPVn%x-slR$d9&7K^BT zaW;tUo{5opdE{!<7;!Q%gok1y)X=AaCR|FRoS-&jpOvr{M{d(1e;JrFQOCqemOxZQ zEK~h)I(~FGzzFX)0iW?c3{zZ&@l^(Bd+;gYObCPDhDVG*o-$AJNE=2LAEWDB_>lkC zHM4IUztAr6+wkR)6i;C60y#780K#cTa5KUNGPird!7N4e{1b&H#sWN<%QxwMaVt!} zmIO~s6Y1uGU>s3R!=m`V;5>4Jdv%62=1u*}ShkLVNk|j@+`9T{oI^;O~QE9(Kb^goy;+)wq^%;s!T zkjC6hAx@ChOFBLC5EYc(4iU7L^@cy>LuxZLaaKUPsq`j(-hqwfD3sL+Td($j$CWb1IkgY@)gtjy`y&iH`wUfU_&Jr&1vwv1#G;1M zbgt2{OUQTS6u0`>6F4Cq0ijcDL9%@nofn=9vQZi!{O}>E-}aGh;?F15$%(}Bq8N_2 ztim-*{89O_7m8(Zs8Zq$x^GAsjk*nRt#>l<-c%0P+6}o54jC+W<014o;6_KC(;%!R z1XNp3z^vp$u>WE&5%z2)&$4~VAMfWh)uV*bx$8_e%d5dEhgRlCoiIn`)djTISPt5s z=d+iJ#La$8*?{e*f*@N(3oQ0;f|B*RB-5#o9Jr#bLTI4t4j8jDq}x4S!LGh`;1j<>+ddbtgQSM*KPbhmjyQtO3nEc+ zJR8){MuM~Tc>uW>)?0TBx@OdJkG*w<9kc>j@T4)ZbLl4KQ1ZL&8CV za$O1N+-Zq?m4C>;It6sMo=Uad&*O?Me?VtAnd#L|gvNjgav)*>EBX}KgexXEJO3Rm z`4@mEzQv$!m=NbeYYfTGsE1;!H5fH{hnL#=;=@@T^uGXEMm1KEc6?rmpYpFW2hxnl z?t4a{vWyEh8|r9J(nVM;7RKoN-$LUR1HgFC;Bn2X;cXlleKx?=nSTnvcj zUa@!q11*xw?7409*~l!^bt5<mn5H31ZdD^WQ?(avoOgm3y4NT(TNI8r4U_xVwvwfW8|bl1 zPQ<0-H?&El(&OG2;B1aEzQ}aNuyQ+&2#7(P^-Sn=QGmT_H{jHR$K=D;4WQ#6h8}&V z!P)x=wfnIEB+kFWqYkQIc6vIm#WI?P*uKO{Nj2Ef9nCn{uf%H?YT$F^9+dgFoR@M& z5QgUV;VSR#*migddabO+?&b+<_4+zCS!t3`t4vJ(_T9XMn*_-=*6`W!GCQDj1|9yX zkwQg7wEen{oQQdaSBlnyyrd|z*-3&Uta}-9M`wV@mp|;*CR5}t_XgEp9`t8aGE^G3 z627;ONw=W_lpgRz=jTDVd_@?#%7nrr#UmKH%pPxEmq9WSbW8ALV?&sYRQyYW#nr~3cY$rALpOGj=Uf)xq5ap2>rSQby_iaytxdd z``pZT}l!_v6`RLk?Mvo_;PT< zjuMZ^OW0Lci29m>y!h%{v|UAn9rT`phrd_AU#}u?9{3J=159G(rCr zdoYfZfFEznA=hjL(SN}&QYqSlMb~06&C-E6tIUO|Ee&K{pB(-uqR4b=;B!Ge_TYdV zuiEw^o_uVMGg^Y_NbC;wx$7Jp3gCxLd;c+?PqOsHLv?U`H65hn>d}j)OKVX44$Y=&TG6Zq}{mRQx+XPk)bvd9U#$y2kMs zCZCLg)h-LbRBjH(zGf3>*Q(>=H}04t`-d(p(!s~gJR=ym~iwtXK;5(kJoX8!!53fD5;acA{1J9Z305LPLXNaa)ukNS1Ul zLSv5Rwf8O2s;-rpzHth#ThN7Ah!;Zq&o|g^`Izn*+KI*<4Y+2PATLiN6%!3k(vNb9 zU^~=^g>@<9=CBLt6LVWCSNK96Z_?^_na5R zG;TTyE)v1ouj{GRWZf$+TMVxxZDHZVR@f`<4)31C;n@YdL9s0mGGA-LdsSeJDS`5$o-SKCGEk5k#03Y8}GTJB&J{vO8S2F8Yh~lf|i{Y!#S}2xy0;|--cnJqpG1jXPHtxBO8n7NlG~?mN{2^}4SQXm# zj=F*|St1s+vFQiwfbwlHtoW`u^w2|V!32UcG+ zhO~@)lJran9cuWXVR0zP?C`^!ucx48ofou-P1aKFR8;dl1P@dM;D=Zc&XD8Ml&f|? zuDoZ?@JE8($1*zar$0{Y7Uv{rwd1e%|EQ{2EctuL530^+K*wtxp~Qs5K+H?n7}7N1Pcv128=oDUkmsgMeAHsed6 zZ>E6!qBOGc&nNm&zM3`NppUWZgIIUxGMrdeg=JxdpyToozjp2d<^I~?#G$E z!{xqgmR$^!xaS(Vap@RVZ#qY%ktK1eOGrg)C3*dO89Mi}jEK7COI@z4LVcT8)66V*vH zT~5L6N582ldxKbf>%#A{0#sFqqV&21CJUaSz~^~~Lm&9|$G-r@D^zH%9k#&4FU z{r8r>5DkaUg6|kr%wo`}HdrfJ)7CtFI&XM#{s=uzGGDI8W51FZeVe4oelexTQu#TH z;yY>92Vtte%a(4hH6&RTpGnD$VB#kbOLi|bq1WA788!KBbm#9u{I_jBCw#vt6yNa0 z|B^h3>J>aWyz@rW7#A&4~w(<$WDjR(q`)LED zZxAK&1F+&#G>)5XAkK=CaQ#{=u~{pCRegrMwV}#P3Tp$$qZe>Ct?s1C_LF@!o==Rl zg6Uz%hhuNoPwpPA;wvz6NB#c zQNS5ph!!`-=s{x-FsuloPnMk}Pf|U|^RR`eq_GDsGJfohx(a-L$_>d0D_;7yjZEQ( zLg1eoiZ9;E!p5UrMEo=dwXWU<%?pu~DdOXdY=46D9-OBouV&HqEt>2}*Jbd*cMHes z?+`hac-P!=Llg>p9>v)$$sqn~a&|7(k?K+0QMz>K;vZml?Yu#B-YA-VSgNc&sl>zm~4WH0XX=to7s0E7eh4L zu;`FEFh0THcSaS@Wi{gXCJEk^yC0xmWq>(aFdMV%>!{Swe^^qX47d6DIJb6LV{C*1 zr((|tyTIu=$gIsIrfddj6#57@JFP+PBa*r^jj9UHUfCfu=E1kKwMNTR@U@?n+)6i2;bSD$)Ht$&$=(8Im- z`@A%iq_xnr`Yjrax4@H8|J2})bXd~7d$!PoL0k-^S1vy8G z%xA9rL`5CL@i*52GKCw^SobWb4!}+v!AEjmXmh$q8l?VoKw;lyaLK$1O~S5W zO#69O)vR4Si`bVvyoIKCwku5c-$7Uz#|XGV~NXN2h7jaDF07>lvZ zj?A`w%^3Ttlt}+w2nErPkncn!-VrvTK`TmAQ_3rJ^5g8sTB&!Tm<)Nt*oaDVO;M$2AM)Ld}6l< z_;u9KN+liKZw$cQi=p(Fr#O$Nt&bbZXLBAT$>Y=VK8Rg18yB8e#TO^7AmRRekek01 zctbg6!+S!YSKSA;N+&{m?p*x7E(C0Rgixbbg?3M#lMVJqZ;c-Ola~QGca6|tb_88M z!wgOK1=CkS{3vuYAFLP1W3bOFC}@j^n)hRLE6pc;U#CFAQAaq^ILYSc0Fl~s3|Gul zhdR9@baRj)FZpQ`$saMpj*9p6>)JbJL)~|Az(tbyYUR;0@h&)XAH!^Q>cFLa7Z}H} zFC^L~gV_CN1C~0;ke^=2mM@C}wdG}O&vYZqkZYzk72#ygU<6pOXxemaAqIEX!px!? z{Mo95A<=HIa&{lQn*E)G_W2X*k4e;Ry*PY3IfabRUyfdjv*_rI9VjBR(;RMUkdsDF zKqA{8|`~gjZ4&wKJj~M5TCbT$pE-?L8bfCus zuih>rmYY>Lsi*U3xUnQrdh7(ro20OII1m0P-N&?s$JEIwo6c%hBmr9{Fj?~|Y?OIM zMkn3He;Q6vL8J)s)87LsPH~Q=hz&CdcOCBe_U8dL|xyH1jck zLr_@`q^Sw98|&{cTcD8Rqe1bs~=2y=fPR4 zlTdrhn^bPQh6f(|qI`7%G>PUz-ltHwqgz3)U5kXBs$T42>AA2x&5rUc!!UK@cCcEO z500CZIgj+hCN)AR?$R!zN?OzCqv;ByLT;4IX}?LjM^=HQ_G-fMl;qt>-bdbLS1_x4 z#KBPJCjHd!M*83oy_|p1*ILQ$?uAp^qSc{V!YrcXo-)Jr$#%;g&Y^$YgPpd6mQ@vmzSuZ zPf5TjS4?-0gyP_K{A%n2y$kGNTjeOTYo{>JFu4oUDvlEoS1)R3XbKZIBH7YGWqRLw zHt$n}Jsi=~#KJc%^xQ}|8Z5knf`_;uTd)R3du-wRjqQ{i>%qY=VNS-<4sudToY=gs zp#m=jcem$toH$YRJl(h`b1D7<|3XbTm?672*QG2F}N&Hj3<2Y z8L2)Y0#Y|a|z$V_R#|t+t$swl(=C4OBStX)I6t9VLdMlDZ|GP4?XS)ztMc;t^SLaFQ zl8Gxvj=*f0Y}h*U5!l7GQr!<@^uD4u_BCdbrI}_BxnB@Pjb~D&!-Bk=B@LiqcMsM2 zkx2hMO6KL7F|Kc;81bM$)EU1_H6>0$)|Xt^7|>3i)i+JPM^>n(@d;keGR9h|CN|5w z3X1xKaZZpnW!%=lnK`eZe}x9>>Q_RzdK@)*unTWS`V)f%NqD+slsr$g;cN_vhV17r z*yyrQSaQ+}^#Uv~UNno%y?dP*izvXX_Zf8NbQ!vJ=N;G*Re)k16)676h9}teS($GHBEpNX z{=^5WF;9dXjoylWMdA2m(i5z|a{(T5TgX50HK=oNCxmsIi0u zB54$t|KM@YBu7!nYr zfmCb#6O0pW!e*fdbn4$&+}Br3V}85C;pRyCp)H-tA4sS3YXU(dx&SIgI!Iy9G~y%W zNmQGv!M*(}vFdGv)d?@?r?16$+G+v%NYB89uC0)9&K$Pt&%>$zt%u?ADfEx=6Bu|| zOdWpR#!@q&ixZ3K%aIb?@K+Y}w^u{_)mV%YJO%UD&c=kZ_ql@oQ>eGseeS@rG2*EZ zj9n20pf@zhfrd7Ld!-c)m&c&i?o?1UNWxS73goy~5V&;kQ@K0IP%$-<9tz6G$V4Q{ zl55CAk%gS1F>jnb7C;3Q&CqvnCXLfKqtk46VmtRU3Af%xn~h7MT|^U4Egu1Ah$fQt zdC+s@BlDxf3YUEhrP+a#e#vuyRWi{h&#cUtw8^X=QLR9Cd5z)sC%ZT|TlBH)-wBu> z$-{RFr^z<=H7HTIn0M+y9Njc?9h*1Lg;tH7BtIt3Q_rkurrkh~ypx;3cJ=B}-@9&@ zqN@NRZ{z3=t2$b%Aqb(`!aMm~tokcG^+x~T!a+6Zs&szSGwb(+7`_sB@-XH6$cq6TC_lf}IjTE-%?nYrKtkdGg`7A*hHH z3F`5ROA{FPRoZ}do8X4ZEYRI@97;o4A#Gh0ER3-s_dBOji>CLCf@C!97mo$)<5F;J z&MjJX`U+@xCsTRlBN)W5&)xsa9`#BN0r7Ie;Z+~m(zwIWW4{#l1wJJ_gHxCQzC?c6 zOqg1@knOUc%FMCvCr`6tY4gq95FL08O!y9>z@k3D$z6iVRsTTna1y;Gw**Og<0P&(e!)I>Ela@7+r7=oaTS0|D~ma+t@7BOG*Zvfb}r_{eBw!(+mZ5 zwXn-u2R}N7gO{{A*~E>(T=_t%rXx%Z--`0eOl_#{JQHvj5vN(pgUD>DaT2%x8fB%- zpd+RZS|x6wW?VTvrXb6yDAFSzc#`Zq$fVMSG#|M?D<$zD3$moUSwQBa!iM=;(R@o^`n|X3Ss4ef9V-c7(LipPRptWq~q8 z(Ju^CGgC-%?G2LZv=pzo7Gtx9@0Zg zA?DUT*bo+oYYkKJ;-f*_zg8B}t&)+Rbis)=T|{fr7wrEsMz5bM!NrZ`ux4~RbNJLa zSR1g6#%xVuG30>xIbqEMhl`qXU#8Ccao1b>XhKVt5@inq_-q%TQnB6 z$FMeMQ_<(sZTeR2D(;)il13_Jwr%-{DxE6in0z!TDP!aoMCtmaq3@pFEMmsW0Zhv;kd6(hOw;-#S3*=t?4? zwG+cc*U$@p#E9s_gAlT1FG*P~h5U{uz-z3LcVSZbc1A^~s<&O6;hf+Iw#8j#ldVZ56QZ!|^|LYc%#5 zXN+CBH0EP8u2?7v2Lq)r51cS9&XDqzy5UWJarTH#Ej?;1h%(vynE(1L%oJnEe|KNv zu4%hKmYD*3;5|`YzZ)VsKIRWuWw3noi3&P{czpAd$^Aij-iKd5v4W_s&pm>#=nHHU&b^**kPPz4y70_QcAQ1VX$X*;Y#r%rmCx#O2H`ZhoP6W~CTf2)(vyxVYa z=O1>MA)vWko?ewc4<&URuqY~~*};$Kko`|maZd{kmROK+oof2^1edhDQKx+C46xES z8qC%oCysvG*xRD*^GVHB-QrS=j3;)W%ip6WlG?|ZhsMs2l3?9*G zQhzXbA{sM~oTn8*9qc_nelRh%r#t?hApt|r=?uAr%<$WHMAM%Oc_@fSE=d!0{~d6_ zvKDXG{)5x+Q$SMc72eXIL?~beF7|GrpO0N9XOD)$yU-E3=JFuD6h%qBrxcV82jGU> zE`V}%9Qe-%p06Av(~M_>b^Tmax!i?@_Sd*lvC-U*oa5;5*8{?5IIz3t+M;=rET|^V zfuvo_u(>xAY~!bM=7vV%N=0)T6;(=_KhNYm>oCKN@WU8teFQTcGr(){65Nxw1+NDk zAd4Pl5+nKZP;0?urVpJZbJN<{u&;%&OvbXK@t|OGi{_rYK=^MAqvvFe zohEd0jNC@_k3+noW3))3kAU#|6^hC62;VcGJ3Kgmc_1_Pc>F&%=hH zse3BBMayV% z@1qKu^B0ipglBjzZySg=mBRigB~HMTo3KapH(hWQNNQ{ZU90|>ig*sOW{H>J$Nm=d za0tLH3K{TVX5C~apN}`!w88$iQ}9RrJav0Dn|ChoIa;2bk7ki8Aatt*cs(7V+HD7@ z&NLymYhowOh}9y!3g5ZX>t3KxaiXd1{>b8b@sc zq0kZ_!NP-_HPB${g$D4Q?J6)`L19^J3?^uoLPlF0m~5QP+O5s>dS)ep`#JW_`L{$- z=n9U8-65Dc4_sGkLEMcYvIr&_M$rP^u?uGGfiFm{861wZQ z(}Hn9&S+OFU0+^Der8I7X-y>pY!P=G243H015j4`Vpk+{!| zz$NRb@`n_df4CNt-w2V5L1JjUJC0nO2lwGPO#Guy zM$<CW$5kRM97w+Ewf^J8;647LJ;Ng94@VE2VI` z@P62!+Cf`3iJ`LWT;37MbGW*EGxDvP{O&PE-7BX-`tT~`I2+;P-E+yXU^KqfzXzII z*TL)HWbD8|`0SDaPxKQ|Pdyo@TF#kdhgJB8XN9WW52)%4eu&vp$b3zZr)PHkpbDOs z@v*Nw1UAlu`@>3jc`ORO^qOF((2q=-bTWP0B=3Enj%$YytWzISCvSWBc(o3Eiwc=* zrl}~DJeT)&ixT@;Du9$-l|j`}F`~V>i%1+f3*r(U)S%XcfTDF-hr3phHgijN1*VAfUx6cEeD;4wL}dgBJDQcPg&RJ)*=(+|F}nZ%&d z2s}deVs3pcZRpH{rE6r+XBUNleKFwW!GRvd6s+K8)86;$FtNoCGxAr^%aI)59-WD# a!U>!F-{FVQ+xK&vS%pv(4N+<85=uoRBP-df>{Y^5 zsEjzjpW|$cq*N+JQc_k*(h$Gy!S4<@Zo>1A#<(X>(8%dl24hIkK(LHe9-goH0ZdHy| zTRG+CZKgR&I(R-?1lqaD6cY1TXGMG3?7E+FB^Q%xp&#hYfQMAuAqtY$q|)EpZzj7Kx}q7@OjBVX{iwBn3g&0pRY+j)KUT~F2i2u9oQ|OikY99$$cX! zG{_g@dCTOWz~LO4xWk+=y_~}+Zsg-sXKbLy_QVn+xkorMdxE=gK`9BIQI7NGd*V8u zUqtw|2>#38!tVAG207;`oMKrwDtV*}WUl{0&wgn_XO&VTyBR2dOa;R3Y^AMd`FUk_ zjd-(FAIjn%BVGX@Q4@@~>k2yf>H@pso&dbAPDKxc<&5F01U#9jOoC4?g{lEnvZYBD zmaWvr(DFcbu((P2%BNk`3y+HMy+W4LMwk9OObJHT?z9JjVr{_SF zz%fvJoknjh*#Ue%CSY(R9ItDMf|vFJzSA$}mk*e9`ux@BL*Z4chCyXG&0nDtIr zqszxrjuGc5WR?=m_dBWl%q#HyjTo64R06jnb0A2u0NqdZ5!Ke$kXUpQXNCr#^vd<< z_tg#fGZKi~y-^H|zf8t|1=06I-R!W1Blg%{AqS3x!$+qjShM9hHfd$hDVeLlD&aXT zD?{vZGlfNJl~6QVU>tk83Cn8p>6GQN#MROcyPPD!)ovdy{PNh?tZ6?jz9Z#Dyd;~=;$ z+C=QLBzXgE(GWc5%Pf6)jbu6s!{GB#SpQ4}7a2^&J?(#g&2S_poejslX#<#dsEr)* z8i(<<0-Ogcw_uA?G*#)yf)wX=a-0g0>O-M4r6m=(o1@S~=Q~l)C?|~{-$7mFwA=?h`VI)G7eN`l3zpNPBk2{~Xh3 zY?~j1;mHh0Xy@V9?+(o6fs<>{M0&Gh`!Dy;m?Q2paa(0ybv)+I7fkn98R zn~G4Rp&o=%reM>nd3eHf7tR*k&3G03fw;MMa9W`NPxp8f(dCK5W9r1L3zq@arg$9d z^2Uqpf0zR^-ol;8*-+yx3q}W{(BJ$8DP5O_o#tK88Xkag5~vbU>$^ci%dT;CFO>k1L`B>m~Wb~`m~EsDa&XExb$R zqj>WNzG+&CB2N!vNcale^!Xi)s1K!Qh100xg?p^Ks}%T10Zvy>gGViyc+gi4svTQk zBvXgpm$e|mO~-JNpa51B*F#h$fVXukiJAQa@3`xsWm*R$Y-%C(3xq)6!V2=Xe*wNX zDIv2vZsF;ZO*m?3jMwg(ky(B9=r28mcxD7*xSTWgZn^-MZpbhVu6xm9JrJJ-Tx{aq zBI(>15Z$W^C(zrIzJSPI?v%YuP03M(lJ!a^QEJ^B4G6H zB3QPso%HMNM)j$U7_vDG#5%g^jHM;4k)sv;x@Z~Hf9V4WjXKzED#>g4kwv9<{)e5$ zUeFN{VsvMjER;3er;CKc@ce2QylHS7Z_zv~sN~|ugj0B@IT{o1-X|XAmE@(~eFA$@ zA=@*TJ@#%j7DUYA?0>NrYJSQR$=YJ(m`yS%7`CO>HUgyL@&s8uZh)0)k8$I3L7tU0vm8DwZ{fcFl#V+dRe^lKYovQ!X;xQ0 zC`1-xSZF6JJ97pJiL^zlvTXKA?Q9TV{+Y^_n!*f;`(#*O5N@@lNy)YAq0p<3lDe zQ!f*;@C#H?i$DRM%Qg+XbE+FU6kCYN zJ2Uta%STdr4X6T~G_JPyC$g?*aK3>gY0 zROPrIEx2onXLkGIp#u!4YX+iyb3YL~b)D(^j{{$pER zInsOhsj<>7QgvlM&VO8w0_M}v!KjY9Y8sKF!Ew-QYf9u4HfbqKS=*%doZbC)p#>NdyPa(LSwVYZ9!--m?>i)#6}9oeEJF zNu-0C1UH0Vg6pYjWYd8~P$(mf3tF>L{O1GeJ}-lgyP6T-dSg5n^T1dVQz1jK6*u}% z=gA(=0l7_e*m(9Pb+{u7OVSRpBhsnt6|-a}>z_rOu)cTjJ538iLc?KXA3ub&=WwOn z_hHq%9x`5$Nw03aNgG_u(DU0#@IP&a8oy0&P-!)|G|r_8enadlom1eXd=YY2%;4>5 z7ocZCui-tRyYy%6>$;$_N@{2vimA0x=(5ESF8JJrR~-Vpe71=4?KFb0=&7t_Oe{Xi z{YJg6rW4CGSu`Wuj6ME6lWVUx4NHaB!#%%mbotdFbiX3X39-*%ckEN*`Df(9ynk}x z)0|%N;>se>HxTA5KYkP?y=D-7g)w-VEyD5fs-x>J@NsnfY(UcuVQh{$9Cphj=l0Cv zjM+~|!N6hI&?Ufe@x2V^KP0lIHpj`cWd#^t(?dr;$U{hW6+VkNOP3D)fNXJj`q}0b z*iI7%->57)_l`Wh-l0amnF{c}{Cfyid4Fa5UHrl5=3O%Vs1nt*vLff|Y{-LQF7T}n zVh)~8A{k+Mm{-nWpDERXN9-$-?s6I*t@Xq6Eey8woP_VYwnONgZM^797g+BV(nQ$q zC_44uqpB_?@WJ6RS$lti-f9Ax!@t{%VvwqK-K=yesBB`;D}1MG;wo{46cv-h~^wCIJ*B6ok;E{LZx|hY%!PX zv`+&c2J~=8gdwi{pd8oVKLn%16v50Hfr3eiQ(>SS@dg;{Y z7#jPx)55fTh;N7#RD6ks&1Y7@lcI4dX|N8~1k^EY^#s1{wSn(NkD%vFC3P$o!St{1 zNwsqtI6d`Xger5Oq(~DEirCT{x?+%>*FlpVe8}o;QFtvSn$7Dw39OqL1bmi@EuK~Ml7Gu3l4Rx}~!~TjJ zBxa#ETz+0d6)S9^)nAI%-B`q<{0{6;y*AzYxrGVK3nu=O8aQhM7b;Iiu=ze~Y2Y~m zl5f+DdAdtD;$I!9%A#w3|198vS1#ZvmQD0yF46g$Tu~{b2yg6YC1zn2C>c-$y@h$$ zTck_N|J#9lT~nFVr7w;3*>_~Ywp%C|Vu}xr_(E%{3WSd;@R%hx$Rp)bcr92O%*kN52V!yPI|nAH`URKm9wTzm@gK7TnlWhY?r z_5(Ym4-8`kpoL zC;DHYW|B(4Ak6t=UkEqybUF3S-YD=<8AL5J7+>!$6qX$znKr-a4a?hL z%W0rHU3%y_quaop8O|<$nGA*IFG-4GCyDLxqpk6L&~Sbl6}xMLmZ8ts!smt1qbCHB zHD>U51J5u-=%EYK&9^PHbYMEj&TvtuoQjtAwLv zA&}}JjhCc;gKG6h`t!kBQY>_Xd=>sh4z*ZAE_W4XY}JL4I}+gKKWd<`)`arj*kbC? z2#J(XMZvVgBrU9&t-n)(Pe%@7J!vLY2GcP2n<&msG)18`npG;)vq#Uxf?%d zul$Wt)nBP}MIo))nn-V*D8#b|a_JV>kH@u3>5T=iv7@qxrq0pFUbk=>%%4sc80)da zvevM@s)ibS%0u!*92lvcA-da3xMs>vi4G^5)Tso4x2`o*SlL4ShLwy-$%XiWG48WD?y8w6~OkGfN4${ z5gNJ1=!mh^PB?I zgCyK25{n#*L)7ltXSzo@6fQRJ0Lji>v@Oem6X<=2?&N#Q9t1It^caF~TNH!`4iy>KF`RJzJq} z#ac#qNi0qB6JxSe4e|WFc{HZ*7p;wW$l6+dq*aTifnex*kaNFDH$S(-7t3_PU5g>l zZr4$%IprXAK@3mJOyCn^9h|P*3}Pku^y#nwHVnQrKII+^zW0jR*qzcarFj+Y**y=M zQ}s#mokp6!b)Jp#T}ZWyW{>k-CLyAc=r=YEE_ovjd~y~aC8s0bBW$G*L7)m=8$Sk~V@3+(#!*S@o?N>v|s+-}~! z-Qzv?ZhP(>cfRiz-uvEk@%L@F*L4$hb$4s4=X~hM?$xK?dyi}$f%Y=0f3gB;k*2BU znx2ZE=u`1bro_`}peC3TOsV(^oAea)$jPa(F;7BzW`bogJX6GBs(NI_G}sEB)be^v zrpis~Y>DZON#x9eXw%b8G@5F7G)yKmX{w*98C3jeOqww=Gz~^iMuAPWJtmXHXfiZ5 zqY%?3rfHKjn1QFH(^J7T^qOLiOqdfVk)tL_f;}|Jl*oD~sixF6rYQ8&)YDBgZAqpX zGGd;MO@u?Le~|{!FjGxEOwp4?Jg2F!(5L7?Y91p5$jwv6nvFe7o)FLzLuA!IRQ*Z4 zGBrI-6J*5GG?_9_Bh>Xagv8OH#MIcQrbbUlX^7FOvKmccIKDVkC3O{DTs`boBmTpf7!po^bRMp_QJs>7Qv+@H^8ITMu{m$sE%mRdqG;BXQ<1>^`#ZPCb6Vb zFfUj@0|Xhl)Dze^k1@tsMl0uz8pH=?-8Qh50#RyIl-vt-ZN>|OSgsm!k8Q;3nYbwvi#m2u(2aRzYRE*+ZWr< zI@i$Gu7eXWDZpuvoe}_S8~8q?D1$Pnu|W+>RC2B7>_EXXB}MtSWQjtqe;Cn1>lt$7 zpB{z%sidExH#(e;i?xxX&zODcuyjc;D54O65EEMa)!{o=*>=r~J&KZ($rBr|qM**b zG}n`wezm?`SJZYEvzWFHY2nDU3@jO{S*&z&&r-<8nNGqJy1G}knh&Tg9xot8VxTQeZ&q6W)Ce`bsA1(#|StpAYHCZ1b9?8wZ0bF z>lt)|D>QDPD=qZhEJ#oZ5rJFU2FN4l+ZQ9d7F0cJ{^?l|?IZ1vx;!aXiL#fLsd#K2 zpZ_PlF+dMa4dg!+PtOi&+TDuTW)?)zqY)tZ#tfys$HDB)0LSrv8x zy@spjZh0pZEe|hygna%xvnR~Epfz9=kG3y9+@u%>0xXuf0l@U}dJgI(2sI=`lj+^DvZQxbgEZthZO?dJw%13c>sUs&gx`k7 zWMUm+ciEr_u_A)>sFD8Jah0(uP3#2k{FMox2ujDX+8>VgH12(-fF2+(J8@dce}(Sj9kBR;yM!&nE6(3NnJsLm4&c&H&5>H~)4JFoKD zvs&l(ZUjl1quPBG(qSob{PPv)NXS8sKw7dkK^m+VR1_S z5J;Ui)K%d->8K|zDW4R*%%9u~kAYJI&Dw+cPIqV=1pf`$0AxEAhpz>U0wBgK4lD%^ z^4aU=3o!`ycZ38nYGcXj55-gK&Nw8jAL`pZap2(JOZsNy`uyTjNJ5VYpP&gM+4!Jp zf=EGZQsw?Lk`w-|BmwhqPD}`5I5>C?&A{ZY^1FEGzIEE+VV-sJ!OM0gmleTEW1*}P zl#fUX8CnAzXa(G)cLbP2HSP{4| zbvC1t$-L3=Gi=Dv8j|OtKq3K)3me-+XAU9%26^FcmC&Q$41hs?dgxm}Jf!!jObO6h z7Ji58Xr1#A{NYMqoJXoeoXZ!y6|cm!y?E{zTb*u_8P^*`GQmRn$AgqXN~cS-N7V8fup^5i8Fcp& zO5~ez60P}C3hlA%I-1Vr_^$s(u2o+FVdO&wAV5QgAe`V+ZhS*VWyx&c=lkM^Bdp!hJL{{~>K$ai?Rg_sV*LslnFL6La2gD7Ea<-=lx0_-7KVGCTc zL=9EzO&dkm5G3(%!>QNYs#dZQZrgb^ne$p5Xcq^ zXx>HNtgW2JJokpZGDJM&ZpL{ugBGKyiDI9b{Is6OExD)0(0RRGLwai$+^>g{adXAj8#CZFB`m_nVj=}GFQPz3wC5sX=8O+++#MC4AM3WBYt8%eZ4P}eN48WP znJe?}*p{eAMx5f+tMp$sQ8Q*FqQi@oXmQ!OEPp69rm>JhVO^1$8BRscJN3R^b<)ii@B${BHif7ZRhsdW6K1 zCK-IzqvXjQ^j8(AYvQ!PC-7}jULt0%>M+6$U+F_dknJ8r1y79%M=S0dN^dg;fKBR_ z)Ru8&@>myvaAW^G-NDUAb=J+{X3Ea3N4l!}$eAITi7G71|Bn7G8)R(Iox;ekd}R9t z(TGPWH9eJ28m#59Y+YBTV!)9uAHvaha~(#5@j-a{wQ;C}K}KXYlR$be)Ty9614sRH z+{B*N?tZ-zRDP#FcQ2jV;+*up#9nnIZ58cHm#v%%k4{fJ(ZXST^>>X~T8*-ccHHms z@i6Nagf*qs-aVCc%c->KU+cdADkz#O6BCh@lzANfbxVgd_ znC)CcR^fVi9$Tm%S<%6(_bH&HJ1gVPM1;rk(4va;^vUv1l5+HJeIi;grZjrrdfioh z_bSuBbWZn%^(NE2o=NscG6*UsEFuf-PWLizHD+L6I1f_BewEStwGRBeApOqGB_|8q z|C?gEg>N}8rUDlM9N}fh{{kh)s8Ww_MWWP1iKMy#mR9oPxdiClUkk*Fo1kq(u;>ld z3_+;nV(H|a3%KWC+SR{wv24J^BgzN+hUX7bF*m3zgBGfY5AMlHu>0*WnAlFG6yZk@c|V1 z6)pVV5zcO<_4Bkto-O+G_Tuh z{apcU;Y18_`a)CUH=DXI9&G(oszZ*QAAGBB*R8U=zo#vie?*u;iJHX6yXoqfJ{z=8 z0BqSs2;VF+F-rs$><4H0g_@x(zG`MF5(i(#Vj_q4)hGAZW!P{}R^UI|n}4m}h>ZWz zFf3Kg^{ymN3DwXV9bpoH62aQE9Ezo!2>a{8)hXY_j+U!ouy&+|RuWXg=>2^XAX9?{0?Q0d-m=8TPu+5}A$Ptrd3XWdrRpztt?sd1y+KsEF>8p4tEU&W)O-WBds zrt$P4c8`HIxB(Uw!LwtR(S4~*EuJe* z>^vSQ#82C7m;{=2QzO_1jN4n71VDu$VctRSPsN*V!mNg09s`Vqr@fsW;m%6&VJM2r zsc{Zyjk~mUE>nk=(H_9=G<*B1JLlM)`{dU18?8vW90Gt4Kq*2${mX#<@+1)8ArKG5 ziIBh<8)_QIE;CGX9>^6hLX`ly!gHa@{oEvQl}7(b0q4yAvpY>&UNLu1Io*EG z?OQ1?ifN}j*lHvw_fE%yMyt~34su_xGq{~`yFw;FV>J254|$c8ce}ny)mB_!YWW`Z zy&G%x%Y)`>PozhxM?R~6@GQkx)CgrDCYto>GSXrgynlAThB^XChl#&^jV%HRmL3_1{m-?+hQEbL~c z`l-DoOs?MG~ho)QX za)6G+sEqyxFn6Z*>j=fE#Z%etp7v4!1GX$^q04MwI9S+Lgnb-tBGQcrfx=QPb{dXN zknn2NedYTFR6}y18bb9#d#f&-{B{cBy9UV<;qHBnp?Xi4Tf(9jUR9sL|J z#395U&X&;TvScg81Fh>J5sLIPC?;$oJ2kNUJCwcDKGU$8GxUNX?k>KKfMVOOOz6Z( z!wwjjYVMV}p54sKN1Otl#Q}l3{OB&|uTdOSri00H9ohK@?-VnEYBw2-dccB|F^EQB zBowXkZk>D41$GFLPDVx3e9{HE*R?^miSklV7QCb&I3c!`=FT!Z@o%Q*Lt7|- z!u4x~rBFz~57855@g;W=GqW9AN{E`fHp_@(bR0^kQ2x(o{@mq8;mp6R3V?*euHYPt z8Ajs1C_R|EJ+}?-aT69cRt0}Of1=<#;?seFNUM(NW!xM3g_(>DngRwjaekN0Hh;`9 zGwc3nl@)y<-mscaIby`~;c^9Ez`({=plMqbVc-GW6XCtV(uj?@KT?EvGwc0sq`Hk? zi7-8K>2NR4;s0SN5CPfni~RH7uojl3ytvqmA54IdnYt37uDPP7PE&znA3?OsX$~Ef zLetL+IF&DN$ak2f=i{TPSM*pk5g`DC8j!QY>qcHkfs7iXMGzc#${k$PZ*8pmBE^Mp z^#2b$3w|%tYs%7mB`ec*`vP`UzBZ@Zald}m)E+LQVR=E|1HcZv_DXlT3s-oE*^zXr>9%i5c{HDTPgag|^LN0&(xC@TZoRIFtu&-yso z=%c^vIa?{#j!PAC3pL(<=yk+EFcWO*o{X49|73!15k^hg`Th)-K74te>Z9d$Oj|FlnbujND0^bT`^H?+b4>uP859;n^CpDc!usb$D3KD z1qUvW-l;{j0mFfO81IfFQT6#-0i4kJ9p1O3RGVm>d$?2qU9@Lc+Hp%2#1XL|BJWnG zy7>*tvZL|KAZmZz90aM%yoIVwo<_aoys}ixuqplcs^t)nnOagFi_BNfR7Y+?nblco zQg6hXd&jbz%oM$w&>$prVW~%lHgCZF9SS}sFNC309nI5$p5vPF}9}7e_tmL5vy*vr8 z7dD;{?D?Dp8J=~PudZ8QL1z57C%jURp4kP8E5~IgbR|sC*eacGNoJclae?M@E!Ezl zmk~ycNe>hgg|BFIK%ejZ*@T7qs+CtQ$dH+R_>Q9Icyq-Z>Bl>Hf#3qb zY)XUEgN)VKvQ6XU=3lH~Gs&-8Em7hN(rXaLGreHKia z&~Xl@`{#B9NUmPh;&RG*SFx0iP9@9c;g@j)WxJ>j3r5mU+DOpLtum9UgXBAJBCn;0 zii+Zfz#COo&{62M+eV3Tw`TK=lUwk)pzr{++`meTe5R{t`r|4C5fGs=(eFvDAI_q)8TLS%Q6Fp@6M5L>vnQ zkg4zQ{2c!j$NI^UJm{8inMuc>w=Vsl<)mTxCN(#$Uc~tznGi&gk;L@i9thbzM5`eW zFmkzSB>Y%-Twi34lqC`XVdTX&yAqMbi+L5k?$>9ye;t`nAYzlCT)dk{qB-(l^6Hnc4OF_qdUrhzyP%W3K*T`4j1R{@%Ni6Rz#f4L-o6-!`ng3 zba67J>UnvJ%8fdPElQDRM8J?0C{qy?Ws}$ULW$q)xHs}iHsMHd9Rw2qG!oTz$+_F1 zf=UcjF#2_9L^zOd6CRLd)iRKhP>~C9m7sv3F{KLQ@yMh$a-KuvG2WL}qSc%I5eV3k z*87RwIcU^SSByL47ke_Nb?=MnT$pomX@7=aXZJ}~YcU>U z@MRtLxlvX|GN#BK91&o5$dg@=O5XHja9+t*HJ!;Sf3pAEHhVsbNUR8e()=U`q(|9^ zp_$_!3F8%fh?4z1X`5|5mVa|r1RsB@1NlYlc-P$i1iWV>712z*PI;hI(%R%x*8V?w z6SyU<0!V{zIdQTiob;|)fBLZ1L=`*3=N{yxeVBmpjiMf8fjJ@|P4BF9 zzwR|mxs)3?!LUSTG;&Ovd@JhEI6(*op_^wQpf-rY9WfxV*YlEd#1AV%P#wp$;s8H} zSqu9(%p$=Xz))aiMxOVsi4*VB3-W#UrXrR`nb5i}V9^D^f=`vQ^>q|3{g}aO*aQ%U zaouk-&Q*$Sg4^YV}UHDgo z6V_n!vhniUz{(iK2oqBn$vNFgE^23J2?rcO_3xiOnkIAk>G}sQ(iGlUqF~5g;~mu) zBdDl^LRy7<)pzm<)l&M$@3`YU_y}mtuTqJI9i5#gZEM-V#U8VffDp|RG!PQtryzSF*xTRHbqWXFI~kI|J5q4_0X@L^sla*cxN%K6wYbe{Sp9Gf|v4^q!b{N2_Mr_@<#rm8Z&8-F&#)=UlFv`uVnVS3mvK@h^NWfYIElc zAY3H>>ISkb$IClMhyo@ni{WGENM&cLJHF)H3K#(s*3)&pGPahD`ANcb|At)%D3&64 z0cV+er=_0iiQs0_A?2_`I>Vl=+qF4rW*=MM9uBcuPHX}-~S+?T_*9KZfYdp;7sc6{f1u{{x=F-;1};D{n}F*qJ^o~MTd@Eh zJp#*$95o6(9c~4;rtkQrKLc5Mc#-%Gm`ifmCaFuK-}=9nQ>^ldxoXO9l+jw9r*zFI zIN&yjNX6~O|6*66&~MW5yv%XPzl!i;4l2Mv%@IZEowkrM%-`o%`#gYD*E9QBIaE}0}C}%@XiqZy6MfBPPe9`uL!8CWA9F0uS*eS zqZULh_O-K>DZ;yeW1vB;6eL>gdvbj;J^3eJA?jO=z3P=P1Y6%Q;P9&)3yhKrPB?=d z4n}#*&0?|JmxMzvuVWl0dpt)upL0`4EG}VPra2V17tgRfdWN&AK!r0=iyg-4mA|Mf z7lSBf`VGw-6dG8L9hJ@n(OK+Zgs&jgb!mp;~|?%dH1 zN%!O?y4O4@~m)`r1>3#0%_ZsL? zKH0rLY08P)y1%u3_aD* zqNuqnvC6ZwP3z52m9-@8#J!1H>$K5uKf5Yd@Ux`Z>$f$24upC3>Pw>4lMk>adR`ln7gVe_|1oMuDE;BwC zMKx|$Soa5;c76D45wi~Y{_=w~3&e1yaQ}7C*Br!X*Yqv><`q=-adt7h_if`-rG4Cn zYt{5k@Vy^LJ-%J$Byt>Ajg#{T&py>E<(p^O<;t6)yW~DlzBl(%`$Un%_t&lLB)wzY zr$(1hT8gKUhdz21#`1T_Yofyg2WW`p^!FN|OQ^pLrLQeR%vO%F7}t_&j=toMJ|Ztc zo_8O^waR8OB+EW#NAHH?lA~3D0u!c($Jjz2Z(h+ez5G*j9(Bg@-)V#>{UtlExDKu-n7P{tyqgMCUy?v-3VVEogWZ2PCZe=v|`y&5fA0X`X0!)w`+U z))(1xH+*!B7{VSfW1zf?(GPJ{QA`bUJPAtcPgo*dwCJGhO4BboJjQn{6}mcz+D{_0#d%t;MI~*N=3a zWp8l0J)wtVD^ty=(CiiGZM!mA$$9;cXUrD6*z)<^S+Ul3KMlcdxBY<4IYKyNILaVY z*k$EM+;#gILg?9|g=L_Yc$N(}FIH%{|8STTy}%@s(Jb~!O54tRBqsO)?c$#wHm zl*d()ySiI`i?dr#!~)@-cW#xI9CIb5QM~S~v6oY0#Lw7n6`XhP(Cn^PSu!8Qr6%iK zV9Nc2T6WsUDlEMe8qscesPsf?49G#TR0?g?BaWc6<7ln|r3)6@NY4+r&SAF4-(8l5 zV2B?beNnrlM0qsFme0+2VM)bODD`2YTKf2CdQY$;oTN?MiqFxv zFC4|`79Nr_tEyIV>!xI{q+(t?hjqwu(?g)oee1H1P=tg>Eh{5Ij!yqSIs=UrVYD3K z;P{^KU^ zD#vFh%m_NqaVX?5j%>|{+}{Yy(&ba@J@RVzD=M!e>91)W4^EMM2gt%HON^WM6O1vZ$?rS))+AM#waB zURchrK7Sy32j-o0X|o|y!T{ua;602Gth%ltGdq$4O4S9BWR(3iN-NRJ&Es<4g^=`} z4u~2>G}pm+pQ;hfXgYKu4{8ilzUUY_({_7my!Ha~n+Q9Zw zx=l&W?@bvI3mL*c(iqjIX$WjgB{05e+oN^CJ@3qZyfSFG&BHCdNaYLp3*t zj~AtCmrI&$)zRZj6@-ELe{Scd{V5^jUAKUU!6$@CY5HJx)%vdOB>nYU4)&bR{2)F6 zl!2iPn5(c;-JBFKdKuMr!oZ-Doem`Zr}m$o5Gnl#_gelvrQA``CL2mGa9xtvv*0=K z6n)ksjodX{KehJ1dVk_V%RIr@w$P94p`X;HQnJ7aQF}1LG&R_3S(qb%q_=p$tUryl zFrd@J&C*3-az*P+5(3U=^?Dqo^G^#E3m55FBPvU4Pi6VNYjA6(G7 zZ^rS~AE@r5xBP3B=Gxxk%TM3#aY>hmcfU};D4L(RJ`%?{z-|v0Vwkyor(!pE1jkuB zM7btWkI1&`-G?QDvJhnCpzOinvUiC8Tj45l}YP1Ar^FD#dFc z+}L8ZGx&k~8t%(4F^z$e`8hWj4?-dV!T0z-JVs)C)D$l}eavNe6t2dgp$%=00Z3NcuavZTy0EMF^Xi$h z?!{ZJXfgXhBB(Lckd@q2)-Z=Kkw}G1StVsB;3-<1ratW`jffx2s{L@QMGO*|+$yNz znEN*=QuP6Y;_Y|rB{tYnZzU#TswKXR%ncmvaQyYFOY)3d!6prGghbwld7R$e^w9#`r8<;>K&}@ zKtQ2bX+u;pBH%kR_5V#~9Mf2dFnK`U7wXsbro2V2GckuXF)I4WK^b;Di4tZ44(lbDJ`z<* zxmRf)p=>V>sM{M>f$a?`c48?R*H)@OlPJ zjk8{4Pm)}OjBYX8t2gdU*-YQLXuDb4v;Ie&vGHr7 z8!3%5p$&l<@!C!dPDrMM?pcTjTtHkrB!6i)lr!ZB0hHhI{~Llp>RQT_^paXR{WjB` ziK%*5txOKzMhV@1#|+qGjr5=l0b0%=X}yH1E$6I)1wrTY^;PR)R9QOZ=lyv9Y+t2J&{4Mz0BJZa5W{=nd*pLQvn;t- zs||s`cdJ-(HCqb3aUZiZYw>Yx_Vb4QGhOxBQyC|M4&NpV`k~b9c6Nb^iLwu>KFB_- z9nRiCxBSLYR@1KCJgNgi!2y%++|^az?2H1#L+Z!M@|e|G7`-zw{{6jdF7cXbBb5Tn zeY6{{*k3QhK+WUDKdvM3du(id+sWOuo9V@;9G;+mo?a@ucI;>!0%wh!ZDceB&6_YV zScI@^kbRQxiiEkZBRP^uG~ZW9c5BH3ne9!LOwDy_i`ALW1OeKg&BEsQ*&BIX31oz` zid*kVKaVWNoQU;$tjyP5U-V{>`yH>x2vItQV0`f6x@v(jdz0y$q>I}Xg$Y%Y2{bsE z=}Dth2vxi^jk{Ls)tQRtA)T7XCH@ls%m!kg#t(now_Jhk#DCq#0phTA%cn_cw-OY1 ziPEA%3_Uku=9$c(Xi)v$5m+BPj@6insthE&qjmnXvpI_tAMp?%1`<}A5J$mIny{PA&Rzrm7ji{7 JP>>Eat-5sf*%tr+ diff --git a/tests/v2/fixture/test_format_compatibility/array_12/compressor_2/1 b/tests/v2/fixture/test_format_compatibility/array_12/compressor_2/1 deleted file mode 100644 index cb808fcc9a68dc87d3383ed03a394fac249bebb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6120 zcmVP4T4*sbL0KkKSzYm_-~a{tfB*mg|NsC0|NsC0|NsC0|NsC0|NsC0|NsC0 z|NsC0|Nr0)pK^P*z2ALq>31F1o4W12di#4fdb{fN*L|+`1d<>+3HcD z@0b}>|5FT!g*_TIGeMxzdJ(j0Pba3DOsAyrN0iv7l>Gu~c&CvKfYlzFYGKY7D2SdS)S}gl!6b zfXOGR=^mI<^-oPtQ%#ieo{{N=JX6&2n<=J-RX@lC@+L-^Os2u6gxZfqJtwrI^qCrl z#L@?~H1x?&8b)X`6!h>WO${`srkb9QRLxC_G(AVCJdafMG}>xCPXLMGOq&Gs0%;za zV5g+?CyBIEYJQ{+r}|AM(Unc)pQ(_ZRM|64sUD}QevLIx$*Db)(w?SjPsyr&nKevl zp)~zO#L4P@qgHsBJ0wlWB^6s2D+$O*GM?BMGJ^)G*Qj4FT$82AG&YJth(A zWEwDy02&!I#BD*Lr>T%=&>A!}0Kz>c6Gn{)s(ug=qA@aRZA>PbX{MPlD8otWct)nm zdSyLKOieu}r>LH%qbcGtG)>uV}!~sUrAF_(GUw{AP%>P?oKGXDFWC!YL~5y=LFFSDS@sZl?zi zAAF|6nj|Uoqm+Od!3x!Ri6};YM=_De65QiwB^OF`)h6e+GMozx6qZ9S%LIAgcz2Q1Ge1K927T{Q>0G*V%4BXapHd_6679^)X6S4Ri;4`RqWUVAk}8 z1#3?rktSZeiGLf0B1&wmNr~b@C>=n-V~dF{+77ieB9E|p7Nm1o!ZV99NP42uJDGQ$=+&U(q-X}VI#&New2*O{PC zcrf5(5SC&0SR8>ZP}xC}gWK@j+ZKUT6=fCGn-?{SCnR*|hvL{sM>s7N6^fH(;J0vQ zpwBZN3sB|sNFLRA$XWN(qRY+M<4S7f$XMI|TYN)c9FiiV*QSzEy;(x6`3BQ2AH$dT z=U!*v0|crYU9*rIZm{6R=My|KlXlK^j9}n;O(NrAl?oOnfI^C{apaM?8liZ*x7Wk85>rSms-JkkA`zfom$gK-hy1$;>y89|tUj%}~~A9)H()stU% z7^iWqZV56K1dh@}L_*}p4h zGUdj|G#8-w;6J5BYK$fHDZ=t_r@iJ!se;Z31#KspC2{(VWL%g{QdAqoo z#9$aP6{|?sXn`S?Wq7s%?(zgd^W)Y8N0<1qBlJ4;Q)-UQws;aqcD1yw{y=K#FPM|7 z7iFO1QwSah4<2rdHR3z#YHkF#PMv$dM5io84;3BW_nErh zOQojsBFz*U-d*I#l73~OKatAGq1%7v#o&r_F50LS#Q9uk-(3y)DIAP=>uRa=#pOu! z4@!Z|ux@}|yi*!^)Ec9Etr!*eO?V~zn+%&_sT#Lq!aLrogbG`0+3$4CW5T|-x9`O> z&Y1h!3j0@Xx%*Crpvfi%TcMg}gH_evQBkbRo*z?ZNqs);5bamMXrW_Gx9n8D!FRKz6G7};WJ}U<>Em^WsJ?XaxC`Wm zQSc{0QJu^}Cpr+P!*NL)#!a_exaLA6bgV~5*Oa&F#oV;vrpUXA4;^-!t2`^8;^3V;Y+h~O9WtD z{O3T_aKI#;@yEmhSGSCsq zT5t=UkMWigHtQ8tLTE%J6P#_ej4pk^B@tQ|(~VgNbHci3)~OhAa?;1wY1_SMPgxB3 zA#b4F)Kx49y(UNatbEKRh|cqJSWcfhAQE2ZA~cQoKwjk@?!L!w7viudHQHN!S+rN3 zAJ1Tugc!s_po|^BJ&^-H5IBJWG>!p1&i~xDqaMEZeO9 zCI^t$B)EkIV)!g?FY#)P3jQ1_RHCf~op>i$aUwUX-J7MD9L9GN3I0c_3)m5Ig3wVj zjTkz;{Zm4~l+j#R|7alRx`M8;!&i^n64ALHj!}#)OZp_etq~7^pUDJ3L7s((0IKZ&D%>(3Ql1Z9dc3||^EGy;9EIy|_CK#0<|jAyX>|BFuW z_XcY5+}bP8O!&`rotX4v>rE$9raqql!bICiC*@FaW2outUR^twNu330b)=MUMB2uw zAMMYM?)QJps}0%E_qUQGhD{Tp2?aqvP7)QmI}t}J-H4!qpnPwS{q*xSw)Q*nwuY;R z4ccxO;J;1iY~mZXGp@pDp%`gRDfEq^v$!3^MNIwQ1b}e`$%uSISJC<^i_(k&E}-Ff z@s$XnN*D%Y6WCvvpc^U5z938`?h6v~IBgX{vB>FDUxaqqz6m_WcOmJ_ae7+m!yrC_ z(4Rrx_-6^h0H6s|gHcmAj!|_|tR~INX=4PX;FQ;&6wJQC?OXIkH6~-7nCeX8sMEg? zwnxpZYi?(0@188q#XB#-VxIyAzj&xYHPOd_3SlahcTKTNeMmK+_PtrUg*ZT41mD-CTcR4 zHKf&Hwc?6IsV~DfydTsM!P&t(FM;h0biAepk;h<@F5Bu`GqO@_ih?5_flHL0>0UD|0xeZ&QF1|2J|911AvOfCnG%2F)g+Js0d zQ5bOv{sOY~3}-#v4G@BRCEz8-HX5T3;90_nlZbF~x@&LaZ^?KutJ(}06CL8v)^^xq z)sK;|)D(=7g0B5U6PCJA`DZB?% zr|NedCzND3ZxxEIo~h*|TGTa3&3#59YgPTqj|)UOSA;X`6_TpRaCW6Vd_%12<5UhP z3_jrzK%}}PY-Kz%*BJ;y;?3&)t4}r?QWmh}hyar-+t&Y+#2A2!k1#b>p&%B@#yO9K zcqPDemke+NFBoF$?~-C3pjvS10*%qda0+vvslidjcjPIlRFaPglA&^*hW-lzpXuuU%vxy0e1iA z;8iFDm(%vMWQ8zbsu1~2y{$Wu$QiFw5Mu|Lk4>{ZQ9po+P3<*@9b&Qb1Py5jTFdU>~ zRfYXw;tIfAzNU9$ETjCJrzzCoBQ+A!;OUR_A7q<i)IsSZ{)81Z&Sdk&R^%zSh!?I zvMDj09(-g?!e)JFq61SPb7r|r3&4te;?LRU8@1gTE%(US?vp(jjxQtNcZ=h*$>6OQyuv z+cw~vcMm#o?1Fs;ZzM~m*2prkk^P_R+2(Xi9F1Cb^*3RX&KwjE4nFH*+1JWCc^~~) z-i~Mvnttrb>#KrjTGdbYA=A!Sl)8@7Q%?QX;WN3~L6r$ZO70{De8_Yxwm4Wh6pSG< zo#Zp)=y2U%&0cUdcuOy9aw^@=xK#CfQ}N0>hZ9?hYshK+;ME3xdheqHK`x3)Q#fiP zt+tw52N@yk68St^Cr`4?Vj)=y^NB?D*?gH#%06>;`pUKjd8S%vEr*%!6)FpZ>^?kr zW1CHI6XO+pm}n@w24_+7ZqEF%watvNd_HVWL|EmgI#b%YX5%}?I5QXM6N3MRL9tfx zzoWP!-j|=pU~7{T=n_}AS)j^hGXg4pPtO_UP$EN9ry!{F?@G1tR%n`8Xo{+E!V{;_ z#4#1pM?eBNLej;ZQn4~i8>wM`M?=X`(b2?=BPtDlL3|#~NlG+Vb`|8^OFBO@x>F@Z zWixTc()K=FRiB=;J!*q!XR(?OWp=<|Io7wry^w*rbZO2{BvnmR#Hej}AT6ivfN7Rte_`(*|1`I99OD=U{kKvA1lfAUyN8P^_Vq! z4Tz>dVhDNm?NPEM9hSSsG9vwqIUyIb#5jrL>TmAzhKG3e(rwq27!|j&>vjM{|+uIvK`Fja%beo9wOYb6_8(N$kh@io0=mS#oB{i9cG?U zy<(q-R$vp}mq%WzkFOnS=UMxloq&Xx`-uo;-1Ay?D;yI1N{%}TPSi(CtlIIu=1Y_d z+1^Y<)`Ll>t5EvYtEIL~dd_rg@_+#um=zIxn%1ZjZ2m}E=0B#?fjf#eK{1k$Rbphu zgl&4E8me;(g}k~@$xK9(t1?b4U-W=V;t*CTVTllK@Y4tsJ+Y*S@C`dS-C%OtMAtgT zH~JZhiYiX}E^-t@t$Z0+iJ?POA zVavR8IX+rM)?z&&j&zjo4=%ZuqfibCBjb)cho8Vbn@#-a$)8}`XgJYeqL1rVW(7IT z>1g%SyC?1<^Q~9}R6lOdet7!72Pi2(sDr2bY98q)!JnB4S+V$lH-Q-=(Gb0)>E+3*z&R-%xm=u<3DW%HW zcys4Qn`!BIH6<-2<2`jgL4QzNm(3h>^)OU=Hri+T{5XUjbRAw$*(S}7UBpiLKgt_u}zAf z%%8Cv{6N%Kc>84!n=ZDqsDe`6cqduoScHt zxS~FTj*Dgb(rOstB7h&l74KCAmK?(|$5oijOq7>bqsx5+Wl{}oFc-Q1$PfmPOn|_h z-h&QI9bH#!;O)5Z8%d&xiG;LgWnb|bhdTmgX%D|svdF~VunSUZBkUd-f_`m=KUT=W zNFSX*CybF^H}Y!Cv2ll9r517Jl4?9>n$CT}nMo1Y9BS>I`2%4c%i@-fcbcfQLgzEM zzO)z_9IPE7mN#gg1)mhjcvt3-!Tha+HD6LYYzH$L(0U+ zKIEA`oRjL%0A?b;E77Iy-;ap=Ga?V$%yJi-G(1jZ!U8<`Xk!{owWW;S&Uv<~a6^pj)8CZ(%_;fkO zDPiGL80Udi1n?+n~<#o?hN%J#=P#uB<**8&v@7F3ga7_zx`fi7rPkiC#JZ6mYsSzoE z5|a2u#%PdjS(*IQCWC#3sPf;cFnK3aDeq8v7DiCy z+lcYmP>4*5<01|0&$P$cflGOe5+I(kN{PPg(|o5I8vE2~$p_Y&y=;+1e*r1xrGL54 zz%S{m($aR;;NEv0SlxI^Q0O?L2Qrr~44xs0tYb^P_FQh3o^@6IFT|^x*iD*o8l?Sc zQIHRqlIBiJWbAoH7dZ`z4sMgT`v}^g%p(kYH(x7(2V*khcbJimi>$1et#F|H)2#e^ zbIcdvV$i9r0bHzfLq-#Od_y3cZ_6X9%TVDxju{Y*K**L)3?Kdp33%V7sNq@k+<9ey zx7TTrWDn2`;g6-1+mt|;+n{w6E9okDB4aOH@l@;zo&d)LVo~fa7`$|c5)AYtQx|vq zDL`V_GI8axwc}n{{1mpDH5d4)jQQpl_Hf%$R z4$R}?MM)}bJ;QZobvtys@SAQs0r&#^;&f7(kt0YU^)T4l{*{!=t;pEXFs>f_ zr|0Vqpf=@m80fn#ty6brR`saV*m zFx04-o&1RaIPO|a`YHX6qf(hU6x!7g5^ z1ISM}Lz4+>M|cyE=u0u{+k|4R;{i3koC)p4Ps#D)RAV-57j$c012^vf@TOeD7cH8J z5*jX~356fA&3XZ{L+|mO@CMMoZX#|dbk$u4 z+mptr)5$sUqe1Z(B6WTeaFm@WEA^O@2>e7679HrKi(5_gK3j;izibTSLn2{}0q~gc zH7Q>YLxh_90Mz^qHxNHFG>qd=5_s)a^#*#MY}f<`WS~|n9wp3&m@sdL>j0Lpxdb}y zz%V=H(kfQ2WaB`%0G;esByc?uyes*yr-JWTwB4r~%3Qp)G9SpvOF-}K~q|}A3I>I@BRbLoJ#^F2^P%SsumazQ zk1w;IQiq+uuyD)y{Goe?UXLfM*@_;R6)5lbG>0 zt_iWGYr8<00XL}Fnrf1|DnqAtfD7}eB}(6;5&IedjQR4}z_Za`EEBAanFK- zc#cFv6KBN&anKHQnvw3pW7TTC(r$_^L@{ZHCwqR5iPGaeu<#VvdcS9fm{w#Y)tEM` z^Og{(ddU(Ue;OomOF#viLM)J*0r2xcAsOD}38|QdkDBKJLrXj$>FEt+XbIy$5e?gv zGX0YqfBzcG;{}^+9@lYA{`3*&wm(4l>}1Y^a!@h0=^(U0EFp3y7eRjJ#ex4~y>qrX zM~?!5NfY4W>Jq}M&g!T|pK-d`bO*W!A)L)Za=?}{winKkw0xnX4&eh9tah%>cxxr&~7gr!h8WKWAim^7&Z(}!H0DL zG*-I2kzY03xlV9cJ6U1&Sfhx)h4lBhz;)hbLPu7Cw&e$GA(agS2fY)>P(BSt;6r$u zyNHXhx%0{VuzTVC$Rly5)@AM9o+X}zrKW#Jf9T6g!7pH<$0<3_c7rfV7X^LtZzhI5 z&b9TssF4}hK(X<`5Pd(!VQ8%vb}oqZIcsH!^|k>J-cm=&KEc59wXS+y=~DlbaaGt# ztaZkLHuron{4vAWevP^?PotTuFIdoYTpF43+fmV5)^gcviYva$7z0nZB*7oCDsi4K zv7fr9eI^9*ESylmc5&S{4g``eirVLy4`P2pVG(b1l)ZEH&Dn%*9Koc}i@6Q(Wk^o< z4uc$@szLcjngqLR77rR2$b0Pv5ndyU)9$;002YSS&Ib`g$5~=V&fo=$eI?h`|2Aam z%>^SK2vw&WMk)D(o2~~FB8&sYi9M4Y1~f0=surRdsc&=I_6098dq;@}o;SnGi)j?v zN33;kS@TYCj}Y8j?;lY=i`aY^Bicu>`NkzZjA>5mmIk zc%_Gz(JHeY#}h>irOzd*Gn)(2TA;jN{xRy?QZi-!+Dg-&4`|oG$8Ik3!d$q3ZMs z-eSMYjT6_if#FV0vv}2C?DJp|cpS`$L6zLG}uNxSJOYu%(x_mToIPWo6$?vl{Y)|=Ai}cWoM&t$LzQaGdX4&iZ6bNW0*c; zmg{J)-7JWj|8x<}IbsVR$Y0lwg7NSzJDo=($Wk zRY%g%#AY5ajc0S2PylSxMWv5{X?-PC%%1UyOv7SLeG!%%7i2L*u8rdJ0X$mzNvQb6 zDhgf%#fe`)5@361=@$YCd(v#y{43(#>)4J`VS^(GZQ)dG zsQqQo;4TCFLytop zO%3k?19h{-%MHfs$RjDCyvmKlSxZ%zdyp8tWh>UpMr%F8YSNyf7z}nK^zLUMg}F_S zA6Hd-`z{|=-E;%}qXID-#gZHcD?_%O(5Nt18RYaI16p*18%3Qcg4G+88Sx<}3g-=M z9=`6%mk$x%S|@>)Cxhg9EpT1lh{V=7fIl3S3YyNcYWm|X5FXgZss1fRPDY~4x8$hm z$k|IvUKnyHYE*OYiO}#-py891AQG;zDdI|@oeyR=XrG=<{(;OFvX!M=*K)+i_@@K# zt4`v0CL4FoS0_6di_M3H>U&EaDdgHTx}T@`no=I|7AnR3VQ!WeUHH)r{C?bXsCr)_ z?t38V^O!X8Pq87yjU-y%hfI*Cf~6np)k!ajB0#gYhUrf#xIQ9GDGbF4_yWrKIAiO9 z_w91?gBrxZjwsU5yJ+UZAMCXFR@;mj>O<@{Fr{?Qi0BR1Y|2|u2b3525%u93 zKAQFg2Pe7`<^2NKxTY3OnL`-J@t6$#ZZx2L=&6wTPTt`so|2xy9~2$u*hJshYO?o2 zvpkM##r_o%_6&T(K8X7Eg;QJM9w7^K-dXGWK-%zSg;+aD-05yfTHYgAxrZ=B(2ze& zxU5_CR=aEa*EXz|M5%Dnq8MIVH{l^1W$j-uGy4pPst>DxHv{N9`%E()!artGWF|{F zk8PqW6QNm^`xpSe0vNV_v`Zzd6UN+qzEZo=^2bA{xbk|Havp`_$E_lN$SD#a@XxX* z@f6$5_YvHPiltPA+*&&4a;J`8UY?9PX6fR;%1Zesz(d?JNQQljqtO>HVQ9%JT%64; zZ{KL*wXqnuzhz-Ivjxe^@k+EMayOI~n!1xSOBy|iw4bOsC!M?h5-DF8*F6F!T zGw0xo$((gSo+$eYn8!Vvc?u7KrkFFdu+pSOSRXEd?YJj&$B$U^@JnPhJkjFXY;<(% zk)}ypF@fwC2u{*BUZKAFT%hp44nI3+|f4`6L|Lw9Q_uGuf{FmKiEq)HPH;U^>+E3vC=7NFN3d zAsWaDMY(XpH&DppH_U_e2 zYu>By=JhC8xup>?^41;_{Y9nCGkEzuov7MZ(Lm;%B8D`jZ&;3RyXI$1meMkL3CyrL^;MA;~ct!#ZL`|`KYwE zU%P}S_cb&=>>@|T{x(OG z>(~))$QL*+p^?rt0ay+Bz|Y06#%n7f#@;|ll~?w#5H@gz_jSHVA3y*OkGFf~ElabWK#kdjyu|j-6O>kPW55m% zHjZl|Gi)Dbs@%ttmCfN0jWadq{ofi&USjskFJYT<&A-nd-3tECwYPaMgW+Fk@R~`D zmTWF(IOxX)Mh>9Lb}8>6>`OV@%#)UIRp;*`O*VPc-xE5F4Hs{;pv1#k0Qm}J5IE*e zjK3M<-v9{~XPrcPKGg-|2S5zHh}ZM0*Z`14XWiZrMc$ju5nKk78*h1JR?oFDoU9bf z*XT=|1#LNOOxQ7-ZA_XcQ3J~CB{oJ_NK2CYrOd}hC5C-W1V0`b z1^aW$1{bukA!qTSeZ!B&EiNMPdp%?PW=V;&uxt7NrbBdNPpi*iXmuK2D*tZ8ko!4E z^_o(&&v~+_F(V(HYexmId!pil1ylV!gc$2k{-5+Pm5B zQ2XM2bZmJYh&Dg)!GUZb6pe>LwQR8{Lro#B%dfna9t_;zJd|eh&z!Vr5>rAwhgzx} z$2%4N0WNwfKw`L#&BN{Zql^`ASXDu=A7X(twMlcUw>;#O*r?hEzBoyG?2LkC4W^hFNGGB*ET(G?8me)-k>sE{9FkHrnOW;8>my=aaHOZ!r72acx2#32*}~AKj7TQ4c#}gg~b@B z%xTysw{f3BqWY)W)7y+7`#q@$zhKM`_vsP$k{aque{SqmE+8q~B%ZOs?P{1oZ zf!p1#4L;x#Kxs9AC`ulHv4I}I0?Er{y1bADhmA%y!Yq@Ku~ld?cn>qW2}L#d3Ij*J zMH{VSE`{zsB?U9kSc|sW3=I<|BU z&RZz(zS^927M)gqRt8RdWwZA#l#Dhb-IM#MF}jH$fZ;(gq}x0A^8p}588r{GQFrIZCBx<^`5*#FN0FNdB2Dl-vIF9Kxdn- z-K6voP*lH1%0M`(F&O!w*7h<$iv$B(KS-|dLqwz{RA2^3+`R#>qz;iTNo z5j;lE_#NGB3NB|11v>0QB{aqkG=JX0TL@j_2+NNJ7r2ZxpMTXlBwV72nE#zCv5!at zom0Z2{5AGOUv(wTR^;Xn)uY^gTef~oc8t8mg{@zr0%<~D^WF$F9%?7nJ-JwS2r@w3 zMI8Gi%mnPOtHWJD*Y3ZNZg$@9fv5Z33^UkSqAdU^=~ond%sm!cFhHm?8Y$W~vTVLL z7egQA()dLzCU-Ou^@X6kKPLL-9d=W_X4lFy18JY{NKXH5SrI3hquT&-al}b(B>sU5 z3!Dbd;2ZuxbW#ylPJ~9oclJuzoQe*%e|dh9GH9DqiWP6db!Gzq<@F3;0=$AN5DtbT z(p}paJOQD)ry8sAoL{)?D2WwZ^>$2uiG6d@RVFPLRHp^lYHHaXQO_iNa=p?88&ZYQ zhJ@9!1zB78o0-DRiJRgL(FE8^a#(+r=kS|kh}wE&nrOtLEv|s*$?qaXInF27hZ$Ht zmls?gM1qW_-939$N8&*tHfWu3hL1Wl9Oo3t^B+|Xbnq_|x1c2+m-%@4umW0rD+cL9 zZ^YV!QI#JxGPXHqz*ob_J8w;_;XW@4G0$fu?}ElTuO7Dk1b_dK`GL+CL|; zV(KmUq|vS7C9cCrhPN~G@_?In4}$Q8-cj(hwa7sI20A;gg*ib_+5vCg9A+NH2bDL% zId(QMs845=72U@OZ#zkv!@%hlWp3fZo;6_M* z*h2#f?8q#WpBeZo2RULnPx2ssSf16-&3NE`pM-N1D3ngv$01y%2u1sL*7!v#ER82P z&qp4-(k6ddD$p;;2cU1i+NR*I$MML-*)vecYefd~EP8WQG{5BeSdS4yA6gCPk1c}N=zfe3|r4CI(%ZdK= ziSi*-TpGz2kX<0ujIJT;=|Nr;=mFs_fBKT^AEtulU%pAdG{QxT@WcAV0z*DA4$JuK z$A?~$V;~z!`tYipDO&+9k^`N{)o!_5_K=}0_pz-Yhf-qEDHx56f#kxvf|Qt3=&jX5 zVi9QHqELM_;B^a#O4^%JCEiczQvX@KhLt*`Ip)@wi{Rk+e`4dk;W2_I5Na{cF_CaV zAV73zl@a&ja_N{ntDZ<|^ma^CJdhBgnvRI6<Pm3U0eRcanq0j=pWkw-|Hho&$IA zgw#EBIyt0)Fh7ZSW8;>YbQC*juEpwc+C`w&;m9i%bvTp`4C6ao)hNiXJOLe)K@jTbIh^!k^EjC%qjs$9mm z>Zpf%lhWw?9x^0elCJWrNIBwCPMp}iEp)hmRu~QkZ1b_YpMR$$*k9bO&9^-x!bzN3 zTnLcu5e0DW#LBTJikHR-F!5eyZ1g2Ez__8QuYNMb!GAz8e(;Apjl^nTQ>v+LCaI5~ zB+f5d?hqq=UJSumD&C zm=VT(GPv!^QD^w+_aYO8NgWe`Z>%4brER5(6Y^1 z6!}^MZ0FAiO+L5mx0-!Bg+W2m)|s52QGw{RRbZ*DGEFrqK$nSWv7AI zto)JM-7g~-zU(Zavj5eKsaKsajd+!{`vC9l@oyP<7#HFA^QV7hH|G zLG0fwrH%-j=~L?X-iV6KF^A;5&EAyFm!j*x2t?R{2hd*YNTo+!o_X6Ry2s<2_k-Np zy>$EZIdx^;he6CIutao4ArM~YbnAU5c)u!l#|~makuOM+=wmaYJW~z^f8c?k#(`H)7>pS$)9V%le z7EYWCROcNyV(r;OoZZXBz`SGE@r|hUc&kqb4s~IPXZq28>xnIH0K}%gHFM#hE=Ig4 zmDT+qcf)7mdVZ&kntvtCbOvaIwC5DVPoi$pCv@rBM!;x!EO@K07OKC%3$cdNZ8Sd=!0gt|f}4B@OL7H_Ym^ zOdIAzq*2$vp@J^@<$KJYo5un%`8KRDc_!W$9yT`Tpt2IJB8xwijosecbD(`ySa}bM zBO8-|&-cCX5cU~F-A&S@c%xk6_y!5&SgBG?<4$OLDJW9MIIwko+lab%j_h~|40N4U zx58rrrqHCdy1pbueg@;p=Rn5Di-z0EYzPNga1Odha@(kxuM-dw^0q4s& zvZG}L>dN6B2WC3DEf19?i#)~vtD{iD;awiaTtkHzTg7OXN9^F{84QrowXt#@#FAH2 z2I%s&AYz(xLlHMaM#PhJ5&0mLCJ*6Obvq#ybVMSFa|1LZ4!S4!Al>-YE{*6D5Z3-I z!}uS_rSlWmR9}iL;vzyk_!u~1e!^~r>*7(_SayHB8gHUr0w7`U0;fIz$hOB}`;i0a z_}dV2ZRRdhtR45`<2PLVXj%_ay2qF4hkktOyr3>~(uXVK1dKWF=mkX9+_dx4A6xv4 zPHiu7B=g_B@Q_)8b(IhUbgR$lJI!5RwVM0E-yWaDo5TyK>ba2}lkXvSw8xaNv5gYH z$eJGt-G%_uR#MCKW8*VF`b75#!~#5{lhr3)HStKcF0S{o&&Wr`F@S(_Bv)6S5QkSM zFu~-t*1}Jrg^<_qQT1H5O5a1F<~@K`cpN}3#tDd&zNbU$mSxi5a5@n_NOZy8M`-Tl ztGU}`sQ5Y&x?x+^A#E#NirT|fq(^{z@i%=XTq+IQ^Ni_zy+0*vnQbNYJ zvdOv9_{AF``}NA*1v`PNfIUVa_TbIYy$09i6Qy{WBTP2xsvCuRFCcT$O)pLQxM1ga z6XRq+;jt1J;d|I?;t_t6ybdEH2TI{M_4)tq=!q)*I z_dh?Qend6)X}-4f6bz`IVAgZn9zlD-QZEORB%@D3Gi+B{-!Kkv=8mm;dmdjxUJ(}Y z=OE4WES148CdgDry~))=-tcDNRT3}h4fL*%LyxM^<_aWV8~O8nA2~?+N{%WsjyHc% z$sn8673&i@bon7{qX&E46!_%y}3H zKbIi*^$Tu@Jrq40;{#LNb`GO6P6HOsVJ!^#1y4x*w15(iJo@l?XmQ^PH^zA=e$Bgb zJDZ8qg&k<de88?jEfIo!n~3wB@q6u@07)c~XqS zpYe2?0HE_ebE#T~Y=oB*kT&jOJ=|FAM!$BOhv zM7oX~@x|ve>c#yAG>2bfsJl@_artKv3~nKa@5?4O_#kE^{R|E=VWcBl^oUtq|HHxJ zcukac5ntzbjb8eUK>;$F9<%Qg++k13`h=72%Is0pRatK~$bNck@uGM7KY4Y$wX95c z)@Y(HV|#es6Bqxu*yORw#)Ns6SbCX{(8uAW%ico}e+x&@2Gc|y+o8zkknGM(8Ok1U zY{Vl8@9bPKD01Fs?!It>>NUUo&9RRO^o`vCvN+7ta~Nr9$=3#2#a@vAaCzb%IGFRu zw8(cs=JzcCBmYbGNw`24R_!@BxWSMnyr5r#@Bl9sZfr%N7 z0HDXALHiitEn-5$ATf}$tf%d<^a)VO`~a5#tvdmV*MYI}PN43zmKr{f_tS&qQ1u?< zv2}Y`*u8#O-0#H=9Rx;~-Ad=+3FPJI{Y2!a1Oa(tmKvVMw8)!^>F^PFPFjeEi^us( z7vn)`ELV7C21Numl-)dS1I>ti#?|FEcSvU0z0jS*h2#?k^Zl710s9qe@yR$@{FWgB z+JXZKK7@vWf2N9RR3fwd!_SHz>8ahIO4{?q1(CJ9f`g*jIR^znjsdv$Ip=DAVk=SZ$MD}PVsncTh*iULAcwU9IpUmv>98W-3T;VzlVOQfQa`db?AY@=* zHplUZ;=>@)IHcE!H&Ye{OCqBQ=NNH(D7-RsEKiX+#9f+>k>Se;?L;37L(O)o(yGNW zzHHJdU>)`V>!Xl_YBFcL`<0J!b8d!t__Wh=2>sYbBA{H$yz?0nE4;|Ki*_=@xY6>? zY?~s4-vc;-6FLUiQ!QY1>KULv`@8^qp^c>d1e>IZwD5btaEeEQ1jFAs5uoLsaO^qBYvX2`?Hprm8z&8; z_ictfXqZfs_#owKW^^`}S20(bmw?rU1>HmjCY{AkH~^W*O9&|OQjUPpb58-kK?U<| zD2E=0>%}&maxweZ40$)tnw@p{_-cOnL3%5khk=AA+)}v3ZCo;N9oLgNfqB3zvr`Bo zw_Qh=inBCX?G2qmc4VESE|q{mdyc`Oube@69;mC|;eyv#>gLiP+|fFPEiZrdBgkW| zVENO~nXc>!upxxecB)qpaTF1#@ebbp7@m0UL`?Q`Yq>a&0xH`IfXdx?mAQfUcAmiO zt1ssIiOn?eeIG2JgE29yrwAHx!%Lpe6Z3so8Zma&l``4uiW99F^l8TkP+n+r&l`J$ zec$SpEV=>Rh%Ba7{hf0eX*M5Jp;4pw(B~_b@aUd1I<^I#zb6s(qbK+}I@%Q+*$-;b ztE>oc1W^KfP?ILJ9IDY@Hg5e4A~`&#G}wbKVYrwrGoI{lYQr@Ub(^iUr=WM~5Xi9b z19xy?t1-lHk`l3jhYbEu92)((r1htdDgH;(m0g5O-8CEPy|VCj((B;AebQJ*R6b=oBboG&-?9eYl-(5I~<@^oz@ zCPchEXt-rjT<*h9BQA^4e5V=QU-g^avQj+1jyFt09e9#wEHVeC(Lx^JYr#QP9egK_ zAp6$~oI_EG(zK%!~(Y48+83fRtW| ziPF=4s6F6|-#uld>Ae;t*|2ed&)Ej#A*eZOA*PJFQIg63?vV75-j)vIfy{>Oll>M< zriix|M7|v;+4v@d@2-LKI%{v03(+J6;`fg6g};+L+-u$bQy< c81MSi2X+@hb@$%3c%U>6;h>%%KE)y&0KZ9o!2kdN diff --git a/tests/v2/fixture/test_format_compatibility/array_12/compressor_3/1 b/tests/v2/fixture/test_format_compatibility/array_12/compressor_3/1 deleted file mode 100644 index 817b8c5e66543d9f0f7b6974805fc43ae79bf813..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5435 zcmV-B6~yWS0gwdXE&u@FE&u>K6#xJf0000Z6#xJzwJ-f(;4RIl019d~R5cI~?nwty z8YL}IG6&DyqYO!M+{|{eEyN@Zs{-%n4HqlL`MCR63rTXw49W}F^(F4U4UGgf1ug|U z1+PGh=?6g6@ljHquQc`IbB8`$HVl0Sxo!`GM%I?qaDHc%ANzC-myN_byJbNrecy`G z9mPzDm#hME`LOTtid3JnjmCT@hKsCd8#x<+hiKRx8eU__O^0!E;5*WqWIM83XRrvR z3w^+L4a6^=$_wJ(#=gjB()OGK7d5^Y0X@p7oEGt8HRmCqdZZvcZ-Q6zn})pj+EB_e2mB!RL{!AR=fYF#Y#Mr14bdH0 zn|5Ft>Vud7C46&%*LmYyXjK=IUz;QOeLQyjA8wcbu!`aZXf=2qD;iw)immUUHR>Z- zAjovPc^;@*I}UQ(piO&_I3C!>zdmemOexT(>~N<2m?*tP-G)8Ht5bu5nA1uc9Dhv< zg3YHjEBCA@^JRaBTg)m+yy6L%9|f`8D@?z@Q^qiO6&Tn&D9!aTh_PuS2?6#bXy%*Z z7zF#!t#r<(2JY47`k$G(ujm+a3XK6aBbDi20rB|{(!h8h3>LbGS?r^Q;m)wPUN&AD zUQML%?sc8CX&sC<-bQA|YnU+fc6$ZhMx~)|Np;e*H{9Nb5}rf;G4V@00osR)881=* zX_xRar7;_m!asscJ|gk;k_EX*dUFtS$6FEm=B9L)s%y%nYr8 zBsN~4cIR|QM|2KV5?63kk2%U$7A*sW(>I``+B41y#byNAUJu8IXL>->b3cSUg(R>uMIht6RdL=!h2;Bk zZN2x**GW8BvK6;Hxr`e?ha3R*k5O*A((8)v(2@F15R8nE%Ah)dvO`bM#0oYQ8+gJV zP`(K-H0*=-`9N}s7fLGfqO>47)91tZMwM*0z`OCf)FJt^BD^Sura3c%sg`2n{KH$O z{iI7F>|?k4oHImh*hovK0`ss>TR%S%%Loleq}Lf}FJDyx>anOE_`x@$y*h;DFPK5y zjA|)FNv#*+dZJImG=3ry?2}MS;Y6ZbZ_qD$s%V9SQOOAFU}8E^+}AY^ z_;~GFf_G#RxL=8^_%_Z^&18g{ooM0E0ne&-6gwsR%ia;~AV|Ddu$}G=N@}yDq3=WS zs`EVlc&jDI{XtT^HC4yLPA6)&sGOhkFud^#MXYEBKyU5_j<)-_{ZTB|=L*~Uogk?f z2g-Vm2`L_cHvyj~MaLb{RM3407Tw2IP75l5egvOaIn))&ZOqa7+71_Opn&*qB*DHN z7%6Q=4Qbb^VetV(IrSr4By0^T)U<>TS2hA7V)SaA&R4-fc@0loT?CA?FZ?k!2hj@A zo0;Kc@EQ|Gex4%YfF_c&MG0VTE%cs z&ap2F48L36l%E~-aJ0)xw~z$nd8=lALoSXj#ja=%S$6pkrJ$PXcF+m$us*Dgbjv<* zI-Z0x-YekB1JLy9c!Lle)hdmny=eWX$}9VXA`;k8MPJ^xr}tEfYW#JUFWdkCNBQup z$ecKoJG)SP;6|cnTM3WS5A|^5kTi_O1L`__suH6oeh?UnV2=ln(&;ZY-p#1MZ9_rx z^N*RjdH>#TyKRdbPR!`ItYc#?ZS%c<5JD; z9=15T6$ph!!aaINUrNN<_~tHbci!0e4-BcF6*$>q zy9i$KiqeTbb@wbNjBrv>uDcaJdLvc=@4Dsv!5gxDCQ$Sn1IRnk%bI2j(d#Bsu)YzK z9q-A~)Ppq`eX&a#FR{(+y)h>TTcQ0i22}b&4JUnxLiPPl0T_Sj7WEU!4F2_zKpS2F zoTs!#YS}!TaS;qLdI*Rmdnw1(maMY&T0?{$!idVtM0&h1!`laC;p!(q1l$YLXp52X z;|D;g%#WF4Z}z9B2^27zTle_h@-N9JT^%=Div3&cki1h0iC+w}MU1K$<&VUOJ>~5V z9Y%WiSXx?NM-iUikwlEPrJ&KeNVrE&MB(}gRu;{v`KF&VG5o+&(Z8Vs%!@gFyyRZ; zOth5zT7@6ax6@!hASrx?|%T4sp91zs0 zWOD9}fIe!u`8PNXUWVz84l{bPtr~M370uI+=vsUusuWj*8t^VO=R$05vlAd_TXBcS z-W&DXuVOrWRG;3zsKWCTOu76SCBMIkuim8u?Jv$LYDEPCJaTBlCpPN3oRD|F9M8S? z_UDiUq%;s-vKI<7=09<>b}j%<9)KO^{z=`ByUFBjKsnI0P)hto8V0*b*5#Y={4HAs zb+7K+xd_BAtqO;xGn^2x(?IyR6^Cyc?POu4%E21yBGlb%&t^0;uvY>>*IYCbx^ff-> zGJV`zoRZNIoHF`Ur+3azHB%xcVF4fI8mGeaW)4{aH5p4$4lr z)Q`)p3mD#6KfL>`hTTt2mC#kRjA#@Npy;N&RqeMTjWdLfSdK)jPS*2=M%5RIOGblBsu00XmWxJO65%-Nvls z9Ef=20Yy~Mb7zN}Rr1k&u99pso)$h;dFhaxI`UhUG*?vf)LzIeK942Z=FAc6QYy`O zS&A~QM^)ueI}4scD`@jpQgI47K6v1mO&8Lm({ogO`<2Mx77P@yWrrk&bCO71$#qLy z**0qOOk*iHZI zFy=z6r#jOP%IBC%@O^QmZ`PsdQ+Fh|U+$}%BgdE@P{+FoO-IO4uW0%XtH#)92@qfX z)Or~TRzo2`;1$nQ_ueL|OnRa9zm(?CN?^a92vOA~WzTj`iYr;wC#*>&L;nJonx-<~ z!E@;@jfu><{3$2V4CBLukCF$*n=Dx{KSE~CezQN@DacxQ$4ZB1W3%}XJwJYN^io@9 zfZTvi+B6$w)9*PrdpMyfKO+!!2V%(GHbzW%=cEq(H0tVTZ9Ft)WtBE#L&*=}oAn8{ zvEJbTTqtuDeSY|WVzKFP>f!{Nkm2UnWg2?8Kp zq;hV1R*3Tw9niRx2@JCnxJ^wXgN`qp+Pe- zFk!}R#G<~8`r~I@kn%Xz7PXeaHwRs^GKV!KUy_ee@4(SQ#?$b1F)kEc7TQCPjZN*P z7z1<@C0t&&Mmv*{qu?$A7&E@{Dx;rRk!&Q<1ltji&VO>u{6?9Wjr8;AXKo>E9)4E7 z^HI-{zUc1Q&%*Z+!HT9)<~19B`Qb$9*1|Wg931x!=Lc-q4I4nwLpQd)a4-bB{%uoU zHp43X1(IQh#?6I!9;^6nQkEtV7xXb`NC@v}WqE@lShOsHFdhkj0*_>@8!xFKF+*<9 z3-=Updpsi&c9|JYT*D6L=U{CRH=3a_fAukLM+;(l(Ma4b6Dj2f1;u6ofqc$Y;=3#7 z$?*-V7H2XC#J51KJs@b}?}4{AXC_|g7Em<1iH?&$vX^7utfCkbh>G+eOlf|@O3QpZ zBWWinUGoY{w%?^j>c4Q*xs5V#|FuTCm2@z&8#j4%tb)#0;Wi|OT-9?M6}bW5@Av!t ze!t)E_xt^RzrT1W<#|xK+b=i7b9QI^Uyq(9@})5cVo3BROl4yzLI-ZSI^MHmM;vfg z!>6{kz2FAh7cbxxQ~D&>mOxj$Gpfa`fRNUS#yt2RJ6Yn%Sdm zq0UM?bp= zclUCzd&NUR&EqR}rX+LrKJ;45RlX&*3%R>+n4{`8(Hi6=PJ{R!6N)Y72!`zdJ>!Q0mC z+I!`vfVjzH9Ju&c6kHhS!47%HjO3hNkzaXYjZQWD)T3iSTgTv*{xj0)R~lB|3T6iX zLzRCqaE1Qi;7JqV?2;#ND0H*2O((mObG)Wd{TQ~=RhYiSmm{upjwdcB>^*cOIPm`L z9N}a}xHA@bfnK!9+kI`apM@ZCS8}rcmIX;RW)+QJ;UIP-3m#8dW63rQ(YYx%hc+2c zp;t4Z%5B;t$eC%z-p`Hh#YXAgjcLXsx%zXszYtqZG>A+jLP?WwlCa0*H~9kvGR@>h z-!rMrZOaaeF8(8WLJ&`m+6DVtt~&2!c;I`=M1GD|r#{uA-y*llUy6>-r+kX~g(A*7 z0sZOR+r&5HGTvPmlC*T|;v7gc*RMh_J_T{4lcBoMX?DoyBs!M*0u(eGjHi>U5gPJC zZ;7s#8N)fTq%>0(DTfOSoIUiud*(|0fKk>zfS30<;0v!q49VVc2JjoxME{JA-qx7I z6#x4Ca~-T$r@TTrTiFx)vL-$|=Le~O3sd)u*g@T@gs8KCQTkyP33Fcou`}W7v18Xt zY|^lqGZ9O&)s!{rTBDtJm7|jx#d@3x%+1T4j)t!!lQ`vuHQ^m7yWyoh69}!azm(>7<333`jHK6Xh?5l_P<7*zh+Op=y-v-G=s`!Z0n&nX;Bb=b-Zy@2 z4=|@I_LQA8 zeH3@&!#~FS>jBmli}C)_?gM9Ja@7S8Tg+*mayMak_RqBSe%RBN&{PT*{sxs%1MXO& z8H0peFU*vGDx$)10H-*f&DN$9oc+8hP&BIPP^s?db%!R1CGzdB(X`KdNh=mq&GYhdchOF71xOH+958FhKiI|avR~t>dI)!M0deM zIcG8A@O3<pE+tXz3(yVuBB?m-CwJz9NOgEHSSy>8@y=Nb$#^3L$oG3Q@-{|C|0%1e lRrI3yBOoZw=NY-7ymb6@m4pwN69V&rAqfEhpw@#ur~;p+Yz+Va diff --git a/tests/v2/fixture/test_format_compatibility/array_12/compressor_4/.zarray b/tests/v2/fixture/test_format_compatibility/array_12/compressor_4/.zarray deleted file mode 100644 index df1f247463..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_12/compressor_4/.zarray +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": { - "blocksize": 0, - "clevel": 1, - "cname": "zstd", - "id": "blosc", - "shuffle": 1 - }, - "dtype": "jfG@h3&ey8nbBuYZ#?5p@zrlUZLq|zl=7c3-YO}z{S z#)awb;M&2sKuLU9C^tH`h3bjsMJ9u9=9>^iv%%C{JQ58K+uj$ciU+~!icH+{!h(7u zQPYZP2G<bw*%+rg*IYLFIM zr>sd27fr@khIT;=A}ZP~8?gf=m+CLU08_07K~Bs?`b9or9h{KBjCGL-%mg6XmY}R9 zG67JmxQ0(u9u~0N=;9!U2vCh8I-FcwLh%yF8_|U%RFY4?A`1Z$7>LeRrqY5`unX#~ zr9z}h5FzamVXX(ycaK1@p-6KgjK)z$k+d(-n$|DCSfRU=VVysqcvo~`JVTfu04z`r z#~_L&+_WX;f*0VVB)=3G1mN&8Yhw%}lhP4eBFJ=ts05e-m_sjI45@j%Tp^*5iwn#w zXKE$H5d#hgt6rD1UV^aB0-yzf+IX^Ds8nX|y!!F+%^KnHjZx;L56WKH6(HuEO=BOA zj=vvAoJ9cvniy^62!=6ZN;K_~1stM?K{Q$^uGE!nPFhHzdOGC`XDRvETOjk)wo8QJ zr4cl^vz}p3WGyT>2m%+SDD-eUE@&u(Z1EdGNfXY<3JZ=VRGFL&9o2c-fTd@uah$*Y~d&o|p*O&hR6i22-RKZq1B9vxMMlHy9E9Oh@cA+Rp3B!G$v9r&9{vm}F;yBANWI}?D-sEkpw0EKZN zu#lm}#i!>3hys(pmWWml2q`&0sev`53$QBJhKyiy(o~8S)e{brP*Xl_3WS=TDwB8w zA>byd+5qCfgfV9nPF&q0Q9#2&ha!M1E@GfK`YR$f2#ln1g##uvvP3!h`Yi(>gjpFP zQ%H5TsD#M)3Ec99dF#c*)C`#|&xpAPiBYyNPato~W<=_0F}_4NSU5CYa7p4BykX}t zt3-&~FPxBhM--?~=x`Fn1;mV}g09@4NiJAJu=rr$V+v9(m#hj{jD*E%5rn=bmAHIO z8*>~-Xh8yqk;_#h6{4m$x;VVWa6sx5=Eh{&+2jDmAunHdPqKQO#0-dez^r@m%80tU zJ4sX?IiUlF&1dl12rX=!<0ZUCFuoiZ^9-mxo-t)i0(2DnVMXR#6VoM0f@cHO6v3Ps zzT-Ow9L9~zZ_0!|G6ncn!w{53+BSh=iftfFg8-s{4Ld8lUt}>EJ)BAix^$6<3!*fW zk1S_OR|K@mGjwz9Q<)e<5VySJqQoc+44o*^FfMcCZOo?0P@CKvj&2Mq7j(1u%)Go& z(H>H9qJxEFz_EveCA3>MU5r@W;8~L?WEjHmC&7v9ThpTCK%h$LT&Je<2dez6|c5%ftoTkKrfG=62rvA)+Pnth5fv( z0*T2XtH7y6k=00{hc8Ftq$WbN2GLFe!f=%UeGc4+(U{XRBeqjPuM8FC!y$@|CZvWUWJeN|CQY(dgoWR> z7PM}_>eQ-?V4`X*tRlvQEHhPjXC9=;>8=9WQo)q^xgz)T1Hsc~ql!7l(M2gkGBm>A zlII0R3oRB(9bzOASUiQCq-OMHh?j9 z1w&GU0#~xV!PzK}PbsWeZFhLkuy&h7HVF#F&r+n~hp`Zom@uy*P+=p8PPh>2FmfsF zNkhom6hsvas~TBGhhhvo2Qpa4w8gGb2|W}d6{0wq>ycOd2RLI!Dnkl}QX%u)&6IBrrxv(~#EI z%Ar)W#m5RNQbYXW&DVtmvoE&V9H7x+*tImc*%q2QRTSf}3?Pt=oTh1wYxogIkXhlF z-X{)*i4-`$Ov%vu)FtxEfTA4pm0+T-KwuX@B~`@Q45lEBnc{fWUWuK?pb16l!3koD4kFJmUNYpsn*rjA67CR9vn`63 z7PwBtB8Jrt7&FZQ3gOo{R^&Aesw|%aIWumIQi&RvFsNiAb>>xp(ZcPO&!mAzDG*9D zX)G~Y6w6IY+gnBP0E6t6rTUtb7L+EVONU)3y#!{Eu7Dzs!4Tla){(d$un$pOm0&(j>?UFH=7cPWioaX!(FZl5Vz-kyrQ#S}@!Hsc< z-)mchHP6}_Q8G+o5IdGS9*lg*Fw;b{(uJ{;(QkFEb`F-pZsSlFAZQ~*`&z$V{A^f@ zo6n*OJgW_m~uSx zLW-SgxjtU{7paXY~ zOs=b1=mK#D>{)iOdUQ+S>I6WO)HW$d2tm_Wh3V2IcKKs%v)4ltNVuUcxfPN_bZp#s zp$%h$K@jNXE^LPQSEsXubkGpNX4o7~%vz%ML$K$J^2tyu8Jj*BcAv2jlvwuas@6HT z4a?QcEaI}lKmsMP8VIVJWM6C%`Rzt1z-bO;u!k4n1i>!Uh=ptBh)$)hFk%K)BiqP4 z@FJ8L3WKmgm}-?R9C!diLBUy~fysyDlR^sL79&(!_yNH_J0pfVWjv=I-24Rp(sWb? z(F1}a?W>*@6dWYOx-dvmiGpyul7!Zy%$`etqZcSh6~{QwxUAaCBsKc1G&6Pd)pa?= zI)RZTHVDTK2n84^9%N*40|NwV$4_ETS8UG*;tI#aS_>bx6hMK9Ixy*o9{I`$t?Rmi z>q|4{@yKEcwlUeRDuPWoDU&xAY(BmLf$jYo>D}Q?^9Aw@?+T(Ud(M>U^^Prv+uI)( zG7eQeMosKhX`w}TsQ}`K>I=i{ky1%eTHvZLSV$_6H*c#hq=MvMPL4N8R}QdJh7!zj zu)@U3BzKFHFfoxp5W|sSumD&mAOvsdO)-MC@T9;I8l0k>li?4ZCWEV&iI`=gAjNKU zFeG#1;WKqq!LSTV%m`pkRsgHr9Hr8X3PNynz0gBr7*!-lq8S`VW^tPEjSlt*mc2ur zeZ1o>s}cKTv22MCFGFzX4oEVLg~2{ge0kgrGkV>-bdW{{DBEQa$q(_igyL!f0%*d9 zXFX84CV&C8`i9*=jWXcrCF-JXl!AS})y2WV1>_~|Nfj-~*3 zj!^>{+*&49O@&O)6%dNx06&88DwB=y^4w*x!%#)W)B_}Ee4D32B4iR{GkHaT9b24% z>J9uNp-E3blC@7SHeg$XkvMru${b6x8UA4Abq;56S-_DVDwJGPHlByB$6!laol}-D zBbqwvB#b#jy>N54^~1_gPuPnHo~Kx?0svQgv$;*MCIm0VXk59%Ai`#k)h+v*R^h{v zGdxkgSD>s!X@c5Fr1WyjVS_C_GIwKAL*kYmo4DMF4YBxRYKDc(N&yn6%9iBzeDatW zMDZy$<0pmp02U2jRUBTZl%{dJ@S62U#IJ^xsoFFc4g%84rvNX4B5;oMF5L{4%UIkuPri{cwX>?U&vg9rPJ}$`Qfh0_Z83Soi3koF~)aVVFJUi3e zaxA^GLke@9Wo(q_Y9NT=1GG9@h@tPp(gq5iID1BP4}KgIfAkI!qw=AI%Ow)*)EJiu zC4_27-7w{%`~u+24NwTsMv^Nm00^W;;i37(3y=ua$I1jPO(;b$c`~rWMYIJxja{ia za$AHo8T_lDfD+BOBf&JSP3A}?jCINBZIN|>>JfIh$i!w>x=6pjJ3xwi0gTAbS8OhX^#iieF`3*v6D(riPMz?1a5n{U_l0+(oplf`~dG=u_Tkb;Xw;7 zI7w9`Er$h$ddd5u#a$yL3yTE) zMiQb_TzqdduGo;iDz$v5zCvZaguhJj8SyI+_ai5ngL@)($d-~(PO3MyXh4*dl2wN- z4q}f&OhBr*@Q^YIL1gT9a`bKQNv8qCGCO`>JV(O>NP*hSk2vlEu*;!!g zsD|vNAaLNw6L3_*|s!c#1hT4-b` zIQQ+a0ChZMva9xa!S4hJGZ$2HacvNynUaE4lY@-{hk$ICu7G7^K6-99k`(FketiME zc5EWJLP*;Mbs*JfnVGZ0(1`@B-p!%i7&D5FW*(#~_7BXCjE)lg$Xlf&PMhEYOqC}r zEG(jUNK`>V5(;Eigzf{>N!JRYHiCddlMJ<(G^IV*F&d-+TlAv5Q5Cc`x~dq8)I;sV zRUR% z?ghUbiGh=`B* zxU6E<+&YBP7(cgcd}{ zBV}cz`0-vQ`Eq9RfH4EwqDX?!HQB`@(*(hSPycBp4ntctpN5#f(KtlCkAj=v_+wuwSgTi zq+n!^ikGMc5xxoxG@)zI^ebiA$eLML=55DN+67UwBNA#Wv^o}$5onbpX`4Dkh@~!s z*Tx$fM{?M`Xe96;&&-;*0GcXj4y0;?P}1YfSAgBp8eUH~1|!WZ9X6;j%l5ammO+Ej znFdvU>8W$_*TzB%GCO0COsWFmwV6dRGdWUQ+Gh(prbet58!W!EM42G7#m2{vqMju? z(;f_8MmQ*a61n4vpa;snR~0mD?rL>__z(&j73T$)#>o^lUSJ@bKODbux{Od;z{kKA z4e9Og0w_N*x1#AMCxQf{2Q}27#vHsDD}wyh!)laKxl~u>>`1_m8(YfcLaL3B8q-6A zYE5lwZWODVD=iZhSgIyIOMtcrzce^R&?E+d*%t`7EHhZRytX#2GaQD8jvOrwZg@|3 zi?E(b#O;ca@xkG6n&h{0kWUNfrj;r}5-L=%u;Yc95g}t(H;Jm4mKL$iNaH#ohDc`RA85ctD9!SCZ0P-4MGCil!`?fg-QXu$76K6`a{k+*TzqAUsrNLc$Uh zdjho3EG0mzJ9hdMi=hP5-=dW{F?36$P^W?$Y8***UXmGZ;S+vkDcDG5TzRUs?LDAS z;D_oBiYh8T5H4BJDZmug^}`g|Ga@T;h>oa}L06-O>JMDjs4_S9YOv@upmV~AWE`0)MMc&XuE8=9R=xOEsR3;kenCo;lvXAVGd4zrbwUfe1UAwL>Nu+fG83zT zmSYSZF5VXATiDp^VQBg-QT$m)Tr9HXR95fU#YdtDXm(vTZO0+yXjaxoR6F_K4$ z2*4Ymly1EVTEp^y1H)9YK_>yPXHmlzl1I!ghIt0e9E?S}{)I_{>BC82o)qtgH?mXF zS=tpouzV;P`o2P-s)+0cFjb(g?&5@0++C(Zz0Js$s-Foke_ySa=|(QRas7xIi%X#KGy&QdJ&ndcmeV0!F6U*MBpRv z@2d}oGHjeKE;R(N7)1g$fwmYPhlRX}FJy@fcCOE@DO}MH-3E?lWvJB8+aCJEr zLyHlnPSXO7qgur=OIW|`5jx=_Kz&f?xGFSxi-bXqCRu=d$Z|u9DE1#U!N2Em-F^~mS;_S{Mlh*Hsow~>$Q!=0=rc$a7u{)z->`YAFc&84_ zbV&uCPHQWZxv(}TB_dF?m1V6WK|+V5P!NOa6)hZ3B4v4Q6;%afVe(l@;p%|ODWHS5 z7%E&)P<^|xIsIc%LJ)9NDa;EIPX@QiE-kBC+#@phHq>@5uMR@0WgM0r>xR(^d^eL! zUd$OF~mwsv0TJ!0{c;n-VXv2#p_NOJQoXvW?U=a5wK;X$N}7GDKC!2}0LZXqBt zLyL<+2!uAl0o^($+Pt8x+v^vkmB60Tn8tL35RMoN11L7mY-Do~o_QtPVqxeL^~X^I zJ}poV7&sfl(wwP{(mu(X%ux*EUViJX1`0p9cJD&x`7D|7;!iu!`fz-Rzc3=%~sJWLmlXgyd=vjcYp5EF*E z0T>jpqu~TXi!Ev_&aPZA`XCZyNSGx+*`R9-10AFitu_I*;5?^8mQx+!4hxG_yufsz zqH9k_wMHEQb0=l+-T>)O2C9Qlr*lxOusrA?^_v}p%n@8fJzVh!D@28=@3AuL1J3E; zk((6(E5aNz*f1PAuux+xOSmjB=`_4vu|+oMe#!BVnOchaDh@9W4#CCV8V6VWV@w zhgPMi<3~{|G}FjXWM+`busOddgk`>qcqI-+pQc!l(ivUl;xe`xYdNEb-r(1ZN0N$D z0Ow1R?$`B|(uL$&l^mr-o+i^&G1SBoTZwL_g&8AX4pNXEkDIMTP}rO{LAIUY67_hu z^Q5N`hr*7XBuZo^xm-Xi6SbgNP{Gqn=L7+Ni8rM_vO2aR%q)nF0JcJ%9Z(YVnu|fJ z<9k*YE7-QB2pMG~@jC=bff_*ro<96uibzcSS~9Fs0OU5rCo~061qwraj|6P`{V8FJ z_}h?RPB7FDGtmwR-3$e}v5=h(JU~w3qR?{J=V+nJiW=?}8^1#bTv19e5d|+Nl%bOL zNLG>3D*!I`?!|4A5Mp9t0(ptylOMrIluVLb2B`sEE`4^5CFpuxq2*Wai>_tYQIjVU zBPJsN@L9wYgX=a`YRcdP0a>*^2Q?YXBE9I4mDw(Vju^@EwVLI^35K^w1AzI9IK>Fl zhVLs-E7BJ4!BS3N0>!$%N)QXim=53?9k}xGvjW-F;ERcN-7!`pI6ET;v9KyoC=IU^ zip#-l?+znYP?@a;Zb3yKWOlwX@MKLh9LovB3kq{XiR(eqOJaj6S0Pqt$fD$lE$@aZ zkeXML6zpyAvD38pMCbvoqM6D~FI*WkmYhK|`^f=hbPMfKs^%2SjZ`#1E}LUY7-5S9 zYX|&d(FHF|`VCbp%W@k90hP-#<0FnGbee+TXtCo*UzZt}$SPWVOId~~M`qZlb}@ZK z0p_I2PlUt_=egoB!84E$tUA|;{p9+A~3fjK>3Maz#W6CW6?Wo;g%g~Or- z-se1?8hxl>En)o}0VROZcksyWhb<@rIhxc6775c5`Dt<(a|^vY!65F&UfeH(!NQ(A zA7x2yJwuSP)$bD0S_r>q!T@CcFvSw}DLB&Us|=wRWtHM{iVrNUa$YcBpIF{5O0wO^ zfXRnBHA_{dlyzeWq=7I`x9ke1%?ZuwNRkcn8BaMouLZgxT)PbUx@4K@%-VP@E@p_q z;)+aa76W)?9?C!p*E3=Xkx`^#fg62+Sv5scZi)MbR)`rIv!-Z9U)QD#DR87jS1B-z z?WKgO{9J|QS;KX-1^Eu zSEZBE5zsXY!kA4q_JG3`oEal@Dr!yf26kMy8S4`~3gWQHQ6qx{SEVVAI6aHh6uH%b zl%OHSAc4G2oZ}P~32CK4r2!oPc$Nf&b`*qbeE8tN^jl(y=IH0%6)BfT08-pc6_RO~ z#;^*QHw{?%9u!p(bu^Pklv4j2fFQMdv#bVeda+z%BUj)bBL9D%@{+1ha^atb`Ui`~e8p!d3vCwG6~TAQZvs!w8Pg>8H)!0v}1C0AaH?2VQoW0?45B zOdP(snmK^YurnZvlO+}jEqEpzc_X(=u`D>Pa0-J|Xe|XrVQW-XoZ*O*N`grhO*R0o zVrjHUhMh)8i05WNFtMh2m&kMaRK#}b%$rjO1XP1Cc!b$NiTiXAgYwepG*I?MDTAu4 zPP{lS8Di(=KNC2o1DxhGHV8)Wf$b==dfG*)Vl+eBGB^klPjs=U)EV$-YD0;Lq5=Y4 zI!rohF`J~nn$@mNKoSBNuU24u2AEaYVTe>vRzz|HlnmSYqN6|< zVsn_;X9}36q9rgDb z!_kbI1`xbFK-5q?k&fImkpsR=)syb1Fewr9M4%Ph zRKN<6za6JyEv+CEkcedMDlMQE<`m{(>3XVQMaDBZ0vw$YOVoT0eE21h3tcn)EzBWE zDd-}oG}A}yl&leb51tIe93W9s3#eJuA!<*9&w>a#bX=R(I-%IP-m;L<_vpqVG7FRu zOG!w04v#{QBIcGQPnHzS0-Mm_F~AVV=ENvsL z!tGKqz_9als7NR5V`^{K?_~kX2BL3qU1|o$XbEbyS)R~RdSZox53@#^3%GBVYh4JB zcv5E;yEsHMCp31lArMoxGj+rjxVA=Ln87RtSN=AE7)}5K#DrJK?S&nAVvdNIF(b?%W84bY^LGotf!)JW2UjuTMW!;=$+X%G!_v?v_9VV4Hf$v*6W zq(u}6U@2REkVaOpeu*rW@iAe5BLvG8(Hb35nqZ`-c=e>D63umwQ%{D1E_zO?cw|-* zUgjw#&h`*!t?L2?FZ6{V=7%g|q_`9%EX<<(8BP+NQgN zkU@rx41r^Zx62Cv7?Mp#uIY~p557NwJ}`Q1{-AT*%@V>jt$uk`?c_=%ehBaTfBWoO z|JJ@gynQ^@Klc6pueFco{aT*)YoBL-z5nGsZ`b|Hr?-_I?-Z z_xD+A*W1VMFTZQ8_51AaKMsTWcKv(1{&&5@`~UTq>-qKk`(4)V;{9U1=RMx;Kku!# zhuvkcy?y_Fe|e8*->-kXzw~|&Yxn&w{|;|?|GU=yA9kPrx4eJWvTObC{~y-wKl_Wt z@9%nV*ZW<+YkBMK^L}?N`+nKCe%aq2Ec|-b{#$STde5%+e&71t|Mv#l@3U|H|9bDe z9c=#n@4w}L-{QTs*8k(Z#pCz)z4yC#%UZu{*ZZ~0?mvI{_s_5QZ>`;5`~JQCf4`Pp z_TR2w{{MG(|MCB8{rlZryx)KS{@2>I-Y@>~)?2&3|NIZOKkU9;yMEvE{_XF3*5R$U z@A2NR{r|W7`}W^^`}TL=TAp3MexKi8{M)ztmbZ3!tbe?;1FO?pn()|JMJtUyrxHc6Ytq-+%vp z|KI!l-Tlkm|NX;1-tO=3^6c-gKkR$VVf?E2mJ`~3P{Z};C?miLEUi+ykR z?LND9-(c_l{@eHLTD#Bx`v0}-f9ItpB(B*8W?&|K2};Saw-{5AW^oF1z?qW&iSj->!YXZ~Z&$ z^7ysu|M&OT@A~K8uiwMJ<*l{Wd++dm`StJZ|Gsy4)<6FL+5PpGZ+ZLw@7LS4&%XU- zf9-YWtul=>w`?Wla{r$)8@Bc54eJuXHJ^Ze>-v7T{cK7$$b-~F}w`}qB} z{QvHD|6~7mch~xT|M-8u>#_V`vj6t?{r>OzUG|rM|9jZ|H`sWK z_w28K|GWS1-&%IPwfp?;TlSy#|9Jj??ef2V{r~U3eY^PG$6ITc-Tmi1{@4Fu-@a$p zul?WkZ+H1$`_J#&^~@d1Ywg?L{e8Rt?|1G0KKA#2|KGBUeQ(*r z{<3d(E&rDH|GN(V@1JMwvj2Yl|L#BU{Vw*|-T%endAo1lzPAqhSkGI#`+v>;VcE63 zwbt(L`)9H1|GoeJwVt*6c6afMKOFx5m)(E7-QQii>)H1Xe|W!#W$nLTzyG^$mtXAv z+kcP$-QD%r_t)|6hD)V;UA`}@3K|G(bu z?yq%NetB=-;_WYM-!E(bFED@q|9{{A?*6`gZ{NSSlbB!xi&}xOggYn7#&F>tx->zs0ye(-yY09pC*krO1M&RZg(WGVv;T~;829~7u8|FD4+}(VLGTW zD7rMLK+I4ZgmR73Op-weP7oV_4yv6vd+=mQM2r)DiKptkQb7I+#?zsK*#1O&cX+qX-xiBlQ?DAR$bkMWF^)j|UtHdSu~(wpZH;ODi5C z+(_~0At+VSs$`@x!8mv=F+$ZQCCx#mj^S0YYR>FW4SL zCl70$m-G#Xpb(e!o7OaP*fF)~=ayew_DvZC9EYR!fK>1nu15QOD9%*0;dwlQo6d>J z&Ku;`qV(ZXilFLAoUOBzukfA24Umv)eGM^g&sUKC1=Q9+h?VP)0@ca?yE4)6Jd&Hh UT?4*EbSI$4ZUoZ(Ew9V}0h#z%wg3PC diff --git a/tests/v2/fixture/test_format_compatibility/array_12/compressor_4/1 b/tests/v2/fixture/test_format_compatibility/array_12/compressor_4/1 deleted file mode 100644 index f1721ae903b7f29cbf894a3552762aaf58e0161c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5380 zcmV+f75nM}0g(jYE&u@FE&u=o6#xJf0002&6aWAywJ-f(;4KxW0IEl2PcIM^B8H1u_L!PE(HP z3C25qz>vADc@av$IK8o9TCjC@hes&E6KFvkmtdVFj*)K*4Cd);j@QT(Lddnx-0>tP z2P2Iay`a3ZHc8HqrL5^y zoY@N{g7@@{1`X(&taRDED6vopfythc!c4^0lX4L6hU4I69}XsA*xbl)Ln@etCgX4c z3@&vHkVjB>@v-@8x;7LX>2!`}GuCJtP;eEj9e`u;n2X7f7#Wz_2?MzJ&TOioifN*A zNyuvXj9jwdiBv)ZvxS0<3z!o%PD-vjii$04*n+k+Qg)yq&7{DV7P-)t=@5)+40q|6blx&^CCq}H4btw6J84-{*V zQhiEXNw{H}_3&9EPSTTuWubg}^LPBmB(QSDanM*Sj}F;njfp#DIGG*JGZzquhmedF zUq^3)Q`~H0gVWY!l7~BZfIqN}q2!#RZ$il+Tp1M!juAWJKw^_&K zW09)d*0rpYB-a^EWDm!d1C+U1?X~6HGwI|KBt^nYMYjM*h7~BpOd$e8YAq4ROPn!_ zjtq{iKb=6IO5a1lM-8n;$6Xg9i&bT0rVUEDHtlLs-!|qOJ3bbx3$5=!gj`-x-`NcuV@j zt83?nYX~~A93`-`n)>8#>*C96tVM%cXTU&Y^o3qzs5k6`PxqKZU!(G!S%%ufTTys zXDId1f?8XhB9z&v7GDciCgfMNYFu%ws?L@P9z{!vY@1~%&_?E55Sw8b6=K2Zuw_B( zm`i}hYHEcXPc$x+8wEtNrbAT8YM*DR6PAG-!RYLu%|HyOAi}9%2&p(9NW?PykG>eS z4eXinayT`gJQo5uhZ*oYa6Dy)3S`BILKiiI(icMy@@5xmjwxXHw#vkj=|v0mY+A=X zajD)K9z>~)<0;OSN&?0pprwH%ooa87u&&xSsYk=?f;1gOT4?Z&ZKWtsv!_EXbPp%W z(XF$>hpn`diB1|MBXWMIB0j;ICehKGV9BnI9>GRbw2d`GRurBb-jL1vOJs8(;O|c? z)CkDfg(f#RiuE8JCs>};-IWrF5r@4B%@m3mxReqTA{is=0A&KZVI)?_yCk|K-$_NP zI33_+8dNLXI;>=fbs-?s zWDiJlMa9>-Du+1=h@b?6c{OIr=USZ2`{rC zsZb))7W}9-mg(fuX__O=$R~+T46l1pog|9rhF~l-U)wrE z1QNq^;2)yE3SLOv=!OAAFz_^-k|l`RvX=otsN-9jFcrq$+v*EMAQmZ3Go1~{EXajv z-oYXkrZiR#dN@fllbtcPCJcroI%DCNwrdShh%|7-1LM`L4=>i}0`Am|NRS8y<=|OP{&Cs1-8+OlA;U!2<96ghbE*3(P~q3U_W@1Xc8?Y6~DP z(ohp#tPqnCN7l$n4Zz`M0E`R<;OZu_wZvJT9?(C0Wiz0}7Dof5ppwg~k(?n_?GiDK zDS}5wo=VxwSfDRIr75f%D}>xSROUKXB}gG+R!PGJ8F@Zk5oy@>tHc?bSO-Sz1rnjg zhvLCNvQw&Rj^a}>ieUgC5-`Mo0`bFd)gA!hHhY)3*|qEEAp(gUH##c9plCuDsE!g2 z03_tauxDckDUo8>9xGU>D!p_VDXZ`@_EJ@{W3X4w?TJ*z?4qfUYH)^CFo>>@#44%C zPg#OoGyzW;WLU^dFt?j?qk(|V0Gzot37?odgse?KPT{%1pkD7hU}HVwnY;;>Ar&IG zN*Kk?StJ|`7v*P)tPW+_9qo|~8;bKQ5y2d?FCZ(_8MvWLp%TZUJj57)J=@Zwqa{$N zN6wFO7b1)pQ@J}LwHkB5)N)HnplU6smsx05>>?Ku$VGvxlTj8qVt2ECNctW@%E2gI zTOS4y#0taOKu5@e%PTAB*Lj=;HdCy6Ji^L4G9KqK$bd|t{H=_ISwi%jB%rCb(HZ)zO8&gqa~3I z=~+d1&?D-NuxW;;Y!NUstp%zhkd4|#7b%w_n0ffhDqf=KI%$LfV5hAJAi_S~0YE4Z zogU=Suw)$W@gTA%jc19+tZo{B$0g|{?6Tr20_WIa%Gn7Y1x!oFNrHT!tRPt}COG0n zq&CslXZx7N-1gZ*frYUZBk$p1V(sow!7JRx-5^ODC1oHx9D=b&6sl_vw!#&PwqPY% z?Z5~KyYQfsHh1EqE{A0YhR1D^Oa+AwIGR&KdeV?obCyEqVAf)&FrDCFY!9%84Tw=~ zDU^A8fsSzn4S~(4tQtcGaW<695Cg^V)}06V2uWj7LpKAO3O%_Q9!?08)|mOwfTuto z&Jd;$RFkQp9}lPx?(yDdY%!mn(E6$_U}>@^thX7ST#P+pH}uV@k=bH(XVvjCxZ~;3r}QMH15Q`2 zu?MqBDvi`BZkYXMRHax-x7rkOnmOEv24%#inwio%7kRi1={dBI2$!sD=)IVQDefjp zjcEck_FRQ9EUe3o`BJ)!iv;3gYQu}KhAC4S+SK(P0;>RZK6X+8p|c|uQFgaT$|1BF zB_a4_J4j+B5s6O+*QOB(2pi-DPUg!kY!{9xoLLVwWLORso!KNveipI;%HbKFoEnvd zA`xXkR!HjgA?m{2=L#Vh357;7o{s`lLD_zj)S1FVwuIm)mC?C57B>0FA#h!RS|Q0b zPw_y26eut>M-fM;if?`S2B_Zz0w(UYXa zY0vLw9-|5r;}rswSl`RM>XOXz;rs)Pi79mX(s$&63guO$S(eIy1&v;0&F=xfGQ(tHBDk zKz##aFW~+R-X-HO0XXDCq)Y_D5%5_bW}OV|QfPoA=b}Rx>W%>iMH*j#*DJIv4A32< z7TVmWB)0t(#V)IfmWML3+qgyo_9As)ki`5$G7=M#ZV>Pig-z;l05ZlQhC9eW0*tG6 z8)nwT<{5b`HRP$lPN@LZF0eafdH?abv6PI_UUX7uwac$yN{DEQpXp>kjA|OiXd#` zl0fh^)ckrZ5UuUR-YjRWB~m1Xkbo_?m4*>ely8Rx3DE?&AfH>@8P>@5U4?QEZV*)k zS_^#f(t6{0yOE&=Oh;38)70wgpqkVXB)Fo?w@(G~V%vkg5I%x8FF|8gR0x$Zn<>y` zkveHuGXrAhPQ}20VpTm5VQ3>X8G~xr{oo<_8?B=nQhpI!rVwd&Y1!FW#KTfRHjyZX z#wKSJWrN;t9ZhmHiZS(Fn#Ghmz_bGqw`;NkACpc|f}zC)kSHjSmirB;*b@mvETK0` zc-sP;X4BPJO{rknO>XbH$eha1ni^L% z2X3s)fuER-8?>dwEa1Ln0QzXlj4m*m&1SRN?1+Od`Na%2l!&_(yeqa7LxYr9z`Yn@C6<4WAwG&lDLL3CgD-n@kkUu zZVim2Dc0@MYn7m;J-5(6wTT;j`jNGK{Mv`sN@CUmA@xZIpo|lWzd%?#228D`wY+rb zw1muNiqQacVo_vniIGnV6VBh`i5~Bjp<8FC=#yHLAsd=%?#^>3g0zx{9*xp%0cNw= zY&M(kzV2$@Z{N}1d)~kI-}ZLz-rxWB{(rmMyz~3pJ#F{T`RAOgZ|-gW{T;=*?|1)F zz4!b3yL;b#x~J3pzSHU7`&akB{=fg*|G)ZqU-!B?!K$}|7*1UJ@;vJ z_jmu?_V?O%-?{gnbI!Z{dH?JFzvp)6y!U?hZg>8D@7?a+`*c_T{?VP=-`jlMKmXm^ z{PVy5_wV=Z)Bpd@`R|-}^;dtj=f3~{zyH1e&%O8j^L~GBcg{WUcVBbwd-weA>HgEa z_n!9Y&VRb|-q&~f`~KCp=l|Nl?*@7KL|e|!J!|G)a*{l3#u{_~&icK`qL&pYq`^WOjGzu(^b z{yG1=`|o|{+;`s5JZ-Og_rLp(zQ6zL&OiVD`*i=d{kz@y`gi+(|NZa(-}cQr?{??D zf4le2?{9y*+noRIIq&|bJO96PyRUiooPTeBukIfG`{(|nd;h-s+w;ypPp^N!zejie zSJ$1Z`ECE+Z~nddcmBKY{qz3Uy`%q6|KIuAcl+P``}FtyfB(I|f4{xmfB$RWzvsXE zbiY6UpQ}IjY1{UH{rUf$`=0*Xckj9Px9{KkwC|mF|NsB}|IR!ApZniE_x%6DeFhiW9;!J%PYvLl&?G&Jh=nph zb@S=R14VPTX*q^;)pi1V@jyjLjU@)8Vi3C|_)%V ziEuz{+(ak0fxckaqXM(*U2ZX2RSG%)Q}~4Uw{q10uhP~QO`~b2?du|;Q!Sz72^9>! zxcV$}fRae;N#%Ny6ol&Z5&DM7$;fGh$;s%T7Mu&YG#m&vZ!c40Q!YWwDVm$@;2=v(`(+jJM0fQ830YYz!vDypuz8 zPMl|YaIGe2H*fHarlwuRLRz_!n?tfzETTK4p@?yB+C}1aPrS6 zrSHy<78fmPn#C9#s&))}#CRAvtmMTO=b?s0QchJR4=EH!d23Y8MDe-+(C3H2C-YW` zu;`ghq>TYNgPvHXxs-z^sTp_zk7ic_C0j`P24NBCn&CzSs}Iqg6^vdU0w_3Tc9z<9 zvWZ%*^%iA8k-TeAo#aQ$H1=cjl}g8&nTj9@87gk5u^3~D5oAP0Mr33(%5ES6LJT2> zya5vEd$<4p|2qoOu3q&d%#vgoLv7M~1BmRgk1eEb4!_g}x8$j~e2*s*j)2O5CcT zZQ`V_{JNg2oh4kIqA2ZVpaq|79n!RnR6i z&ZYWwIfi!~g`P=#!)AL?Ku{T)(WF~OKL#SwV?y+WFoW!cr7-=K*-$4(pEY850}dqA icck{d4&`+h!~Z8t84P5lQJBDJ?CbbV!e`G4_t%yX~HP z;yKTA?%nsVeTIsX4qWF00G^f*7XUyC005|wJ4X{g+$oN@qdAVBdxg&(sdg2Ylg!db)&r&4H~iazS( zYJ8i_rP)WpvA<0Z1wYhE@L}q8jUK6z^`S1$zvK@8)NuX`qm5`@fl zWdKX1P1=uJ?B(5eMquYAntAYI@pSSz?dVFRs7m^P?uU|+%mzlm%OSU|?&I$j+%|M} zbNfVwzT0gKtXsXtzfDIG#Z7e&lVFU1x}6*8x}Y@_o?Y3s;XtJ(eOcve)?^i)BR{@+ zFBc+O<^p{JH&N;o>0%DCO0<*7f%&MGIl=dDO|th*f)`=xTv_@{o*~ zLcd;^8pMt-aAS2O{i1#B^DSp8wA&)E4$UdhHh4#&ER+&fj@1eKR_?X>Z*_t-M1H~A zJq+?~#ak+|bSTHmPS5pO-%^!k=iVo}^W3?eLM%lp#%+x;$3W;vY`~qT9=}eG1QbXq zC3EqyW#rQH?}9}abx1w;9!ui%D1U}wRgS+qaSZhA2h(f9LrLxygIl}~qtvZ_^47<@ zp3z_-(U|$hAK2nHH$nO72h}Kvq0Ajz_N=eyyg+^e6XIJ_;QdD|((AVE`IKh$sMfTDunenk~k^okP* zm@Vy2B9T&AxkW#Tc9II3-o`TYy;`G{KAxQDe?=b3!o4UB&TE_Zs zCR%R?PI_=VW6ecIl|TioeCFtuC_zmWN1=}0)nr7Ejl!eB1Y5JEw#K4xLT8t5 z4CoDn=@DM1V>}rhYxwXP)O{CTYzgV$&ONpw>ynk+64!0`XgDf>^y_ar>@oT39SN#saJx%ji$5Tbzsk5QHi)q^KCW#|wNBzn{5kG?7#hFfz?* zk&F`QUT<5C{->obY+>1S0%mpM<4rQE&52dLZ)25w!ZF~J=ETf-IfE|Mr|IYw^8G_k z#N6+C#ztMdQbffEP+(XVm4Gb0kDg@e zoBP2JS*Y(5D?LnuNP0lZ9Yyb$bOXHzuYz7Vq^kFuVOvu5_U#LPI*~Sz|Hao%Mr1dv zSsxZdF(AMiArWZ5*HpAIOQwO^O@i{LfR~x2+D^;QrpnE0CVW78N28f&1rtm;tHeL2 zvi9aruj1!v8HPT-4aapHlk@fs7y)-Ijx zwUq*xZeTN}i@HdYK)*6CI;!uq?Z5fnS0*m@H-yD~BYsw^TJ?7{uLikGF&NilSR0ZX&C=wElS{ z`>n5L%#k)zH)uHD!NaH^PNluc+$+6NK2HKFfCPji^GXoEr2MT)i`Sq$*@&i) zSn{?*^KRfP%zZJMGwh&*QMPQ-Myni39rIM_DS#w1DKC`d%O*K zqh7B=^+K@4!H<;oN9TQbhn)jU=t(b%+^E0b)P*E{p3D@)4N!nxw5rrVQlxnvE@St$ z#|K;(DbitaljPDD>o)Y~Tclc?gIH?zEH5&RczLbvTloQXJpiaq9C&w`P~nJd@1M}s zGrf6s(gGi`47Bwj!qwKgmTKC|B^s)KvmtH;P^wiI)pyN~s6#Lx<7l6&Nb;MQRv_Tf zn9&mV48!-k%dd0nD%Hvo7od71ghg_iN`uPL_NRsjcX=U=h5DBaD0cpo43=l3qKAnp z=u3W)7uFGNa_S?Crjyj_=N$sEl*>%s>7-KE)tI@vfV5NY<$I?_Z?y06vn0-0t1sd! zJXY|A5{fKM<2lfsHnJ$cKUir8W$iPWHB*zgib!^b(Bjh3;|1+$+@L;aD8*D$o}1Z= z2wv>kHMtD-ELyDzK(XY41pY2oyr9rEbGotq?Ze2f#4hxA z|E@C|q?;AGc`%!5#{1E{613Bz%*6G^yKZ;pGjOd_H0kgUpbr>^mQ^kpw4ON_r56_D zhq3x9<}xyVLW_c)!fw%h=PgT}tT+{g=P5%~t{Mpl@dxEtLlU)-TaJ-s>W8~bi-qwDf9)oJWS}ZSmMW?3@;A^Y*5V;~Vm6XtyF zENB7vNXjRTr_@E*X@I<&(6-j&8GRDCt{2TrSG+Jh{Bb=^eL^-B~FVg5&b)WSa*EH2_OK4FzfK4s5Q)8jL*s0Dpg z@QFxTS>~L5evnn~d^QhGSTs+0r}q~&(xDVNt#`T(+VKUYM39X}er_S!;9VAXR{Tx- z4{m1V4f~wuo@XTSM5uq6FNnPuXg{s#TV9RsV4DnHwp5tf5Q_XQeMc7YVXXNoVvbLA z#Ze6N-Vn_OgRpXZm*gsA^;l3{af0DYO?C$V>=#F#jjd;bVl?J>Wvv4iqW;%(>C13;ALjPu6gdm_ zbmph%(k^8_sP)ev)3gcyB{ZMaps4}3Xe=gt$syio>+Wvr)OoQF+AGZYFK0-7eakFxWloLLgVo4<+1btyf}*$&zvfY~zS#4z zNT&U=*x{-Q)!Y^uN$#Q11BX1Sd`@qL4Gr^wu3r5Ewz#A6g>FsBO)f@j3w)KfeIL>& zRvyq+Qiraz&Yy!9==mU7`woC&qmD>Xa81mmb-?zJHuPxd;l{?bA#f~sLed{|pHRX2%%(k|Ni7DiGR7zn)sJ`pf1Im!qz&MAtK0m1 zKazmTbui=uhUCv-nr~7wHkIFA^dIj<)BUu=*ZPa=!GLXOm;oY~V-33*&-nEM^>wUE zs5wAM1)VsL#o3LTGHz&Rsz(-S-0Co}rSz30MS$fO-$w_@trlaqv{0?WR2l z5M+t7MiHQkFuzwPfzt)XjcZ=!C`u4a3;6c!U=U1a@wk#3$0H?CLk8a2MP-tn86(c= zPKbappc(_<>8f9ye7^JL04etvhU!9*_Sn52VR5zbCu7!S{s1<1EL zN4Cd7Ro1aAGIsuImBr6^M&k}{oNL13`x_ATZI~d-gcgB|orc(Vc;2om4i*<4E4F;Z zq&n^ze5BTA94?kNkua+2N^nfo#I~?b5gCE&E;3Z$)ZxklQ|xLRoOp~$KL{=`7AW0f zkCmA1(sA;+EU~|a^De{QT@dCxw{1EPSC6Q9-nOE}k#Y13cLHtOPLUs64`!1OG^?O&GdUc;4&-{x-mZKS zM!S~m?Qax`Hd;m1XfUJ$gTAm&%ptWM#z9ceY~y-^B@O*$txn(I7&vNsfFLaeW)UAB zx(Z@M8!^}GIWCodCHu-?%%131+7Io;wRRt%f2(~}=l%+_gMxs_Z}}3#HD^hCvBL88 z_^o+Y1ugn>ET&q|ehm^D!v-!Abme#jA79nqO@T*e5tU3Clo zHj)s&Vz>3h10=JX?5FriE31r{v&GwFr;`uNe$~AMzSH>IVfdq3ZKq*%Fv)EU@%k+Z{CzY}nA3Hs1NYexb{k9EKN$F+E+^nZk4Df<__VaLSC& z7exdb(Qxizo8EeJlrwu`*Gf(Z2YB?GqFIxMDuVP~K;n~B1UALwqSo7=$Cj?K7vJh@ z5@hYUSfM1O-M^0n+rO&FoLWR@7&^}GahL`*+`c2r#uyqhs_KP&zQ2`2Gye5R{i0-_ zpy@8p?A@juO5eFq)cUae0Gzdqq&~v5;AYO6YkMX9Qv!8&tixfizk=6*6qrAk49z^L zdYc$C=f@Q{wZpmA;VXPw{>EqVm0Gc!9}Zl)K6nyjH4WmvCRbCR(sagjNKboqslDZj zb9jh~Yx6t)d5Wtv0yuQ#71uV(zCulwx8=25R&@rGx{XeluT zFtL$hV398=bNy=zxGps3ocij#ty9(e?;p)>ZGVw>XLIm*+tFD3tJHX-&u`-VKl-n= zcQXTR_GB<8$SO=JDXH3~P_aU>7=dav6l=5BD+YNZi&iP0$^W(tnMz!;D7@6R(a+Y< zb%6^#0}EHWcSFSvj7k`@L@4Mwus56N=ZuCg%t1J*%|0xP^dDJE9tvt^5I#FtFO#a6 zb{5KYb`=?LU|ma)=Yirc`VVN(mZL@@3fV$+zB+ zMXidqF`)Vkt-Ni?ZBkjsyYseUJb*se3N z-`+Ms-&H$BFr6A++6&`RJ4-It%BjV+71fbM>>i{Zjz_3d4q5!wT%nzQFr~6(e-Cd> z*6ULJQ<3Fbdy>5xK6#legi7At@)|4Z!Z?U^g@xDkpco>J_+k5^cc4xTiSr(yG37?qJp^oX~o7F3DU6<@d z=(x7qGiW$lY$+Tde|wI5z!&TJG7=3 z-|i&rd8PEHdaO6y$6E19=rs1cAwos!x7JRa^A3KxU&*k030ie~UKejFjydK{3`u!8 zn(Ou1pn-sEk(}4R)hW){HZHd=Tou-)_u9Dp!&zH6?QIIINQJ(0{*1I3#)_%&f{RS;omP2MHdK_6>_a>pp~;LDtaAuTkc>LwS7}93r3AO2)~qg zgQx7e+o~-%ou|m$_KI*ZI?Dz=J)O~7;=DB_W4hV~Me_=%b(S+{8-EfDgQ$1$DY@JK;)N`i zMl608BjEbQzy5q!*^kqnWH0M`XmCV&0_6v-i(4~l!L|`6^{k#12|JQ%vWqDmOUK=7 z2wflhkRuA*J+CH}+6_qlLf;@U{-iNTg2wTbRE}>xs4k3Dl7?ZssnD!~+NIs*OZjhK zH|eTjVz(^dlG7WALw?}7;TX8hrDEXUOSrt-yg9|Ag{CT z?NWodlW|vy@?6Q#Vg2QjMz?=+R&=?kg>uK$LS}8HMx!-lziias9L4yy)I89E&(-WS z-AmTZ#pKuu>TwUvXuHd8RkRBQIyU#kBBrkfV&SO;wD|)INk`4;Q*^xnQ-o}mrPKy- z3~Q9{enmg*Iv&dp{rL5cjU@{-q)!c*68GJy197hrWiM@5J01V5@%j}z&?&xlwQ0RV zW5y;}1lbELTDWMu=uvn+n0$Mq&N?AB- z+$ltwCp*c63zFDqJ?c>A${54I=dzdFJ+ibSS<+=dk6liSt&ECY>V3v>`%;Z?1-7*# zjPtTSgka}TAZwNQ`%7JSy5Zcu z0C`MX{N&V}dEHX4HAQ~iotSzgICWZ!guiOT88!ZzI?dESUcZ$A`>;K)#XRz%N}NOY zFug=QAD5IRbI4yW&gN7A!ZJSQ0PEiyCukVGU*7s6$A3e+aHv*XaXo#boT&4vSdW5r zOnuqss*&{(Tb!gkTZzyCF`B@~KRzOYBcC0ZGlY-je;c9S)Zfpwdy$Z2u)#A2Jc<ceQ=p^J+&ea`-5uwGIzKVlx8$3n?_Ra&v3LYPw*Ye_!0ZQ zpN+LD^*eUTabY7PhUu=#bveu1d0L>bLTjG%=xbX`dThgx6y*S%R>GJGnrS7RYd(fp zJi^lD7Kh(#AmyQGocAtiBTO52(68sh%u>TgroZ+^FQe9kam2p!@@_2${S;s`B83WF z=(NcF&@1`9F}vwouvQm%iZ=Ie5J)Z&s7r!908x>uiZwjUFn(umpWhV5O5+^3W@njp zYe;GKt2<7oUzzLX6fcD%r19A{%R=O^n?>o?C}%Cs$rgvQkbaN=6aPolx}B(snh1579BP3W3vmkeb^rYAenm;+up=P@ ztk6S!SNF|hl}lQG?&BQVZ`+M>=rBpy_yTJ87CA$7D@?B`slW}UZ{DF~Fa*cLQ9D`xzhXG8LnQ4bY&@i9%2 zK+W}EUD*X{jOJQs*uDX$A$zsa#&B=VW4N7Dm}5)l&}~7;r7DSZ%ZD3japyW0j|819 zl$U8hfDY`PCS9Oyd9S!`F5~A7*jIP8nA-apEnVIyE0FbpGeI$=eXOtNCDFnugsT@& zI+3+QRrl|WN{@FIe}wg!845bn^B64E@@R1e*{Yhg>wA8o0GWh=@6WX{)0F#Mg&l?D z{#}-?jE7e@cwu2EKkCORd;0NNE_ceTCh2`e7hN8-T4Rz0adk#+9LNP&xs;^NGEEr~ zIOqJ4*WqIqENgPJu6eE>P7%`2P$5>ub&yXbd2(c9)0p@HZB<0aI>xQK@K2omn9u|V zzT3bu&c5$g%<_f+MES#vx<&Cg5hL3kap>^+A&`YlE+MAW5KYS=mYiB4L&4p0L1xEh zj7~-4LL4Z$N0*>!zJ0stiyJ_cu%%fqz>iSEbF0{#VB@C2>1^{~6@U9YR8Us0cUCOx z-=GfZ@xbnH;cWg7;h5-bYAGM(HD_?An%UgrUK`-pKGv8{qxixBYuCVJ(b5PF6_xXj zSGR(uO};JIrIZ<*>Q>h;er#H6;=d-nWgccc%&M@`jPSfC|E{M6rrmxq97xait?Kov z7EM*<`qNPQe|`{pUM!P_E2_3F*RO;#*J&fj%`~9E#!uiE zt-ahtL;EYa!}#`;<0rFw-~7Wrx-lUbkVZaH%`*+{%AmIRa&9ljGOR4 z4bM`Fh55L&JKLc|Uj6X+6!&r$6BgDV7eOE$HW8!;l@N~$jn4n`i2NEc9oC%0Wgj7S zp{>l7(}pIyA5dYUXi`+ZfRzlE%!FB8IHxYR7t^ri@(xdQ3*~}aMc)yN%bF#W&04uA zrSQeB-6E57vAbiO3F>CgoWR7{ODx&=P+N56Wcp`6it(JcuI8|s^Nvz!02)5U0{T*g zzxPMh2O4v;q7|^sfieuK3WcNcZQ)^4M7L43ZYbz1g&r@q+OdG#WpaZiMr<6p*RjE_ zcsL(lYqmO~C{LFz)t9`3;1wOlCI6N?Rl-g|lyPck3qMnjU|}OgU1V6^h z35T{Xz8G}7CWTW>f2<4Z&}+p(W^uG+S(be z>Xa3Ng~>(vnPDT6L_H3~^&W3`)iqbXgiH~YPYh*v(FM*?;w<&}>V+$b46W&!-evtz>2f`xn)ubhmM?tZ{7elj zRp6RKx4=C2T4BK2+`r(%G^h!MY!{!n>K~-AmzE?K}6k-05uTG_RY^7ihushTL)|+pwa~++<%f-f1%H33 zax2V}$L@}$Y{;DztagoF%KZWYm1vScoWNHr8)u#>gwrS{4DrJT9kF8_Z!kp3L41AS)b z@ud9~;u3Lj|JO0UAV^=d=5(=X?N5W*k^fpAL-|r%rx<&y;xj6gsTpqsUzYs4C&0Pb@ah**R9K_3y4 zEjKU(H}nsp;NcMheL_h=cM)B+2&tPuC<1)b27z1EJ}Dr;cija~+E7p+;$|Cqh9rW* zA&?xz9|-&rv5$aU9U@ z0*ttUBP0+|BoQ3E28R0~5R-@C8^lwLa5zNVJlr6}pTH2r$V(ly9ArJv?Mm)k`NL|DhyyFQDLm$4DQ79Cz|51%W;3Kh?qi!I8>0Na{!Ap%C!ErwZ&5iANTZ*e4SBBjmq8>M8Nx z2bo1)4BQG?duqLV()|AjH&;*uyyL(7K^!AKA&^IaAfRi=ZxFFo`-J*$t{mis%pe}& zh^wdE%Ky9Jh%RKX_O!rH^))E?2D$&&b%GQ`fMM6j#$ zNdq_t0m0G$EGz&31;Bu834KZfQIMeErzTVs=^lVI7AgR>5E(qdFi>!wAo55wl9?Fk z0#T7XgbY|fB)c{i;O$cixlm96NF*vi7U?rQt>C9L17R>0az_#oVF7}H*ubX}77!?n z91_{VfD}aqP#_7BAi>pG7?WZPVY?a_#hOCHIX!)faXxCH%;+I`>v@2f38qsB%_Q^r zBw8p=+-Lf$w*DkB7M>O=EzW4+Lh)mAZsXR#e;jEist)6Cmk2VO7H$jTPkjUX<31*w zlP_M`VNNI1sIcJ;y0dWRx|*^Giasebht)-c|Pw=dY7WgFz9WIk)Q+!u~21x@R7 zE(CY;%b4McQ@O`4l;Y-;I~b)4VT477Dd#dY+OF8~I7SNzWo3&1-1x?wO+Cf7M>V%< zURVTS`W(#0m7Q_FF|=~Yj;x-&&7=s!)%N37VKvI<&$@~jZXk!RgK`P@!)}0Xkdia& z@Xf_T=Al22*rlMRubL(roJHLNr$#Ijq-$?oTqlLhg))uYJ65Il*e3lcQ^_O0yszZ1 z$N8{f$m;}F3;LyP{#Sz!&pK0Sw%PuTO?{U%Lm?B@tn*Oxbt}Mes`x!HcsMKDS@=cg zGm64PLB**~R!+C-BJ^GczWmm?XKu{#ncw=w-)1tb>^T95hiG_Z5k4hOV z-uyIAqoq(9o2O`a7IevG^Kx6F?viC4KtC8l&roD&5gROW9kg^R{}zSx;$GC9}s_A05a@P-a&85*T^wuoL)5(jbfW~` zdBn&o+ii9DX*&5qAWGI|p^QfCQVG&d5K6ai!Sh?rIs3`@6#KoLtLH{nZQ>J#Y951R@AN&&yhEZu8E5ct7560gqm!Lft={ zoqTXm*i18HOugxj&0>yMYHxfZ7{7}KAGFk$OjRlUSeCV@lnP>B?&5Rk>Hz@= zRLx9TXmf!rT{uwzUq=!yt@jJvJ0=RE-@-Fm_6ucX_zasSR2cMI<+*1Rr*WF`V|ecr zr4d6%iA`O3HR!Q65ZFcBK<=L+uuS~zPRSYeL>@%2+9D)Sqzj+>>CXx#`Zi(CAqXck z5YN!sd(h-aOxeEs;(&H8G?UaM9v2}{Z56(6u&*tzI+KMO_;_R?TNIE zQ~Wwu3s@odIMS@Kh6mhvxB@}T!5BQHh(I9P_KY82R#C9^AP)9y{IZidtXp&1`2(ky zztLW^WWw92*(T{yA0K^B&|O5fBAf~zI{nr* zA_jrp*30~QVE=+G$iCAt^R45Dpzr^uW`YeX( zwh!;OElt_n`t<9K3FvFW4_iyyK_P|=b6+Y4bI=#+;iFE=y`mZ3n5Bg`@nXjxFA=`G zb2SEBNL1%?N5h80eeYDZxj06>WEA^i3Fv#{{g`I6hYHxKGlbO#v15#PQsbwcY-Qaz z2O-TQXK^~#gTDO!G{jLq-krENRAsfj{5x{bDL|kgSD|Lz%pS_-npR50DAkwQ6PGI& z4PxC0rVy(}x^f_>HaEL2)RHh@0cnTG3f?H(uTlEO;TNt`tjb@#d>7&sXN7|!O^UV!YaRISGDpyC*gP+f2 zBEw?)oU6B!?n2D6_wB*tSef5+hr$&~HVeRJ3vD+T(FN+Z@y}l}fkRM^B6njG2zdcm#r#r|{Z4-=T;U@g?X~8)_0tw)fx~k3 z($izQ9ENd$YgYrKvCVpfv?G3Vb|_&b3}2Z@OiVwEz2FI(!d#P+bv++R+j^<8(E>;b zI)BHmG5vJqim4<=idpge?evNJ+YWk%kOjZi`H+QnUPt86oXQYHo4B$;3^P9*3-?wZ zeeWuX_)qxo^Q)1-&(9xUG&KHPbH#qj_`AEvKQVzJ*q(nIh@AYTe@NS--7zwDZrG@|0`$y1FBX!&5NojI>|knZ!fdQ@!!`=+wS#$psD)i$*BHdLJ}C# z?j`6*%%NM#ZckVne;VdxH$5-QjFNu(X?EF1cst}BM^4ZqOT1P8&uLP`EdCSDx?-2D z6%O3^2`niCA+H;IV22DPWhW!fy$6&dHUIH7S&|%?uJ5gBhUG{&pYa;9`)+~kN>`VK z4o-Z{ulS=5s-Y_L1wkKF8xb!>7lZRUkm(NT0tcj9y=gav7U3E?HQs7ZNV>ccG|{vs z?b)koE8<-;`l*J@P6*8=|Ip2hV9NShDzQFNdz1~gF%D6qp04YRtsFRJa&!#Pe-o_J zl8IDMv{KEiTk8Lvre*kP@@vSi(Wt{HlZq}%hYt=fmti-TdFsuk^SBdQpZ$k|zg>V%pt~cK4y97gP0DY7 zd^Z3VGH^zF%=7xZ31&(E!Q|{t&V-BVDN!x|jpx}WjS@=`OWkA7O7S-lY!k2HGcw0m#Btwm6_MNl>u_amjV&Fj1ZFIk z=`KW3if*y;c*3N7<;1$*!#(xY!Moe(*8EJTQ3nO~>VF!({1y&wp}3yusSa$Fl6c@Z z=)|^IXD%&!pHk1zq)haFfGqS?x0I7&i(L`{72((y*U0`~DH)aT(sSdS6bs_YeZn_L+2^c!ZsrPAUhE63V67|jd znJkpm3jHmGaqey;isZ(}(oYoSgvpjFXIMKmPa_sTO^cX3=GzQX7L=oHV%@v+x(l}t zoZHd0TqaH?<^DWK{)f+Z9L}u%6&w3V9B<9=FDnORB*QJzkK&u?36NDBRyTu#MijR_ z*R(pV?6xL8p!d0G(fLf`eWj}w^%T?c>5UyMczEfo*kkGaTY9&g@i;86VzGn0Nx>gk%!QF|$ zrbTOSllo|fxIG2Us&m>?884a@>my6*lFsLaz?&M;!$vcQ#<)u*l`jHDjd~AFV?xN2 z^hr2uY5|$&TK3W_{d}sCof+kO^?6|%KGf}As;^HPQk0r&MXE|p3XfL;mC6!+o(WzH z1#~RKpa@6leKA$9avAy6`KOutXApd$OQ8i5ce%WZe$_n8aE@gc<2_lO%365h2pv;PxG=(x5!xZf6Fd??)Q`W<6FDjCwSE4^w#V!GqYmMJ0|M&6;qnvD_*DfA+9HMr_K|yddZ87^VdGFm6|cy zLFD1mid}F;S$aFtloU(GKQfQ@$2S=i6!tH_?5n@VdCqkn6U#Hy`Si-X-T zaRLuiaqddJal}&5klFU|3Sp;C&d0>$VbnYoL1>O#giOw6tw2f|{5cE+8C+46%V*1u zryG;CEQfTwQ zWmfF2fCGiINig0sUC6}d4QJRr&Y!hx@+8)LdC%BV7A|=cOB0{nt;w1Uq=YZ6ii}!N zneern>Av=(4WT}K{qgoknx{8G|Dx6K@_r(%JoMo-_e0qleOnz^lYnohMU+;MK2LR| z7QM*)s80PH_~{VY-bnUCoDNSlh;o#+#D%LOE&9+x!A%Ji)>T%m7_=ltP`PaL3o6po z2y-$XWD$Wz^29Ap@grEh(xbTy^D2eE^+Ll*DviqwzaBfl9qo8OWpT&PkXKL}&&vgc zCoo?3hEVg4k??&c*s71|i4qfR?&13LcEK|V*~i4it1}MrwsE}&<2BQfro+g>T@K%H zF-SqYDqC!s?rY(}YicFf`|rk)`0?+?vC6OIbIT3A?+CwKeh0&&Tj$l-oyu;jj@IUUKnttjF~)Y-y*mk?f76L4k$h@eP{tMijto#4jJ& z4o6-1+;5dW4==Q-9CPnQ{@iyBJ{UOeDG&*{Y<9qHO#flPssI?Ex(?X^HF51%#`V=7 zxqbo$toO0ru_pn6;YO{6MPG-k*Y66PjxWz%+JsdXl`jZQEizN~Hb()9D=PD^l^@Gb zHLtwGMI1$8LjLZEjs+&b4u_8SmTOPG_v4nZ>u%Ni?XxFpAs0Ib-#3QM(dO3=vni&B z`*-sJ$7cgw&gcFu`|T7s0Kg60GJwVcxKzwFdIbdpp11?R+E0`_4!|HcE`e(#uz(6r z1Axvo;N}ulta1(9684rOIIyQkG+>LtqQ3)2IACJ~1t6FHc{kYCfy<4evEKD-;QC}5 zz@PzO;h#hNDBKnUpsrCU6o5l-0D#{)aE|Q-fZG5ZfB_z~GQr9AX0v z%%jm5;1IAy153!0C9408?FZ1P@P9i_BX9r?2>{630U)MtEr9?6^RJoPC4j>Lfg8Y5 z@A_>J@UQ4U>%VSpt0VqpV8elnTZBcUZVQosKwxzc!L0(3P)FTL5OpM2RTZQP0!4D( zA`(y}2=7)CfI*!9Fcb_1k#K^bnp|KI7(`7?4f<#Omk$0HdP{gwAaXns?LGUl88eol z2Y#MxFVZ?{txyA2$t;Q5grNU%M2h)s=*da66$BvALGrxWM3{;f@GjYPH zrLJUO%RGmBoSPsTL@b($DpR5;d@9QPNUe;v^>39299e7SAi)_P2~D(VfS z1E&L~!sjJb@&wV$Zr}3n231!ZH$Cs6J6x5jSQI*qbK7!klvQ(2xAeOrkF`L*ySPDe ztLjQ5e5@&6m(ec%S#@z;r^!hjO~XydOnL9esZpj0^?&(ys`_{6z(YvwS>FZ@J2$ct zhvU{)21@aS?qm0USkq-`Y;EnBVgx04;PvtAa66|Mh}vL)HE7%Ll>()u-J80lIzgY)1#Jg;-D&*yy!@T*OEt;fgr|MO+?M7Xs;|N0s9r>GdUmhHq# zUlORcMI!vC96#KCG3>)8)3JqPrc(J#3CHi8b@A_a3Hux8EZJ4>n<- zdK^A-rPyWhlq7nU0f=;an)c5BO7V1#9`%0C8b4;+R(mGg+Tu@GqxPeXnFQ8$rT zx|z49zAoSy75%;$ZR|NHY$ZuzH>lG)e}uu;cQX@mGn0vIl!w{bRR9fja5$j^8lUfg z9Vdi1hjm_)(W`}IU&eIKDjz*wnaDPJ&n^do$9y5*NS$*^wTn?n>Y{=o#q@b+9??vA z1mO#wz@u@JU*sxg8Qd+@;bf-|@@I z09dl?BK>K)mbwb{l8nL|q$zxu%)0m(Ej=XR+qVDEL|`Q-YsEszf@`>Aei13K3xn4F zxAf4_g&=Qf$g7Bt0^jOy=1T=asQUZUuzgE6;I(nG`F$%)Jh_Y6r*%MRp$M(JXM&It2nCMJDb?2g-f;&@ist*d+tR?-0|G>iMH1>SUM>rmgUJmEj7E3*jF7FwV zJY^WsH^SV7d*}n~UF)}ZYT&)@-P6meXP;7)6`ZUmlOKFr7)Q0S{64v6#ZCd0n19K+o zm{`dYh>D11sy|N0j}8YI;oT-8>MCBdTdw6#o~TM{aPh&alS3sec*E)-f;%X`-Kd7s0pgRM1l# z!+&1$u>46FL<oLJg*Pt|w1@?lY~vDx7^~J>Zu92h)W6 zsot8|oJ|VSn42lY39@=gr)M6bg3{X|g4VL$@P~XzZH6Y!3TXW_gQzZj0jXXt;MZ)1 zvyc7(sUzwj`*16j-o(#4u(2G4vN~bw)gJJ;QpPx^_94GoBwlKNgkfi&p=u33r}DWV z=fjCu)KHqvH9B?)`L3MeRzG_JC!`}Fbc!uVwy&b|!gE13N&|!+J|y+qKC(^x`J_5I zkyu_7!x5KNxMqnzDnIr@u`CW%O1we$4Jo5hw*jv8PA1-)%HdkOA=kkngXL~KgdPXn z=%{lVgtdf#YU>G@m3#>HU+g8qo~`6rwlDeP{hX$HlrTDXoylf-HCW})%KWGk=BT{7 zfc6^8LHqN3_EM3!*^en3u>DjJWUFX_#r{oDvR;>DIu+6bcH!*O&)vj%8y_$1co)z=Pu;#X+f=K^++)NuU=rMT4* zN6>jeBub8FgZkM>aJD`VAQ!`W>yAO!j9Tuox6ZJGRv-(WG$wW~-J~4Ke!X%9BMcTp z;FVxVI7m>gD*>H5Es?MC5BXQ8fbP~)skZxhT(RX3=nN+_z1oS;7%)K&L@Z!MpCX%Z z#RO;PzoRAp0`SDQ7}O0D;#_ErA=w%AP;9jZqb6&3sjV+QoYg`93y@`0V-;z~=au*= z|2lIZ&4}#2X9OzCxL~uPj`k#7gw@&)s_ojxq_$`8_Mtq_x z#xKCdfN1U&ix)7^BFW62+eV*_%tBo^f-|xO;af!k#-Fvr!tg8LwEY;Gy>y1%qEmRs zGH$}Rlst6Y=7J+Sdhq#31bL^l2j*nuqjpCEeKD?tC5kb~jZ%Q8p59P*s+2AsYJ|dh zHrUjP@NQ%}bY^B?O5PucP3WgXl@^_^4^YZho_*|%4+Oxo}gB*uVa&yCJD95#N=<^%}cmRkZfZOpB*o= z14?Jm;h!2QR5V1}uj|N(m{)kEXg$bFiZYv>Bsjvlmmzm_28ew5!)|RdMecHMQ2pgW ze?}!krEx3ad;6Gl8!ABQ0Z(*(9)!zRgrTcUC_GX;f}zXo@#cl8kSrcZ-HtWVHLpjg zX1fe8>f8zRVN&6*tQdmDSNtXvh)txHOqo?izLuoWtB3S){^{$;3*wTiXE%e;uS-y; z6@$l{%Rsu%ojr23)BI?o8~k@#40)%;cn`f7L-&bFvc#vBF558|;wv5C!1)GLJy>g= za%(2YE$30TwVn30t>Y!sO2U+BRqUmbN`{?}`0!y1Sr^>~x3)0U z^nfM^z1e|Hdo$T_f!nyX(FQnI=aHnD8Q8vR9r`cvWjuopfTk-8I!;sRqtY^5qJNBt zoEQM(>PP76atyWi%>j8ECE^h}4a4`i5_gU-#JM%IttSaxeP0ef1w_!LBmoqw`8ge_ zZuo*P2Pf<(@rb;HU3G=1uPMljuf9dwRYcf9?7m|bD1@L4sB zEZnt;n6CRw_qNaAg=@?MZe1cAwYr7#29>#~U1z{#MlH>*NrP>L)9L-g!stFV2mHhi z;f6#L^k1^yb_d#aN8m&CI(mvv9ScM9$OGMM+`8*HXv1PQ&9RKm%JAT3-Fi;Nzw`6-_gI+s z8egJo9FJl0$tYOuvH(ow=5Xw5Hi34nIzE2mjybY_=)xi$eB9i_Z1`~-6K-gL_MJK= z*;W+Xjwaymi+;p&O(85^?To>zcVowjrBE+@692vNqK^i+aJy(HT9w~{ln*5|G&mNw zMJa-0Nf#qD=4f7f-x96rTAAq^r|`N3U5JHvA;kZDgYA}&>7JpTXzbB|Yjz3p@-$L0 z(cmQgD3=JfLycHimqKn1yO2JC7ofaI3f_KDfvrMDaPq@Vwt0e>T|S!)^}n@l7S8jnd$=Arn0=i^Ihq^KrR$ z5L)lxM-8Vks86q>DgEM{x?h64Lm%(p;Mps%|7tUwpSB%mDMT|e_DjL%jR$S?nMHP~ z7@14vuz2J|IUP?~Z60CJ#+U?lqwE+5Ceo*nw{l^~TD^nwM=Pk@;y60pN*%sxjzWgh zWQ{-V02)gIa74b6S-(OQUoBq@Uxn5}vBVQtr6$HpIH-!TUWKr6&vn#*^)R9t4?pG) zabw1+(6)D+m8!0TbAN9`TF*wpS?G(c!9Bn?-yUk6_G96WYsB+&D&ZHJhmvNMBs?`5 zeqBwaaYYy4__fz4H!i`^8TMcroX$hVtDVI4sXZ3`lVPLw|A#83`e5R*io|MJ(hubv z_**fHS-fR7xNh}=CntdD}K4G(=9| zfoDFj`l2zUW#p5jXF}*u!v_tELqTSTALe{L1ug5mphavlmTIS>n(rZapdtW2#DZ{! z9G9kCwF7eHJ#&UX66`*f(Rn}pabmYPCqb(nf4%=lRn20_-$Q;-bw&e9K1g$pw|`-u zpWcc8ez?#dZTWO#PbBLYavI;&Si|vdmUQ0D#j}r3z;f0W1NNq(TW2LRh5rqf4$DF8 zwI#gyjlrmx)eP_MEkgJqfX7AhVb0Ihc_f{k&S;o(TDQYtmy`Qj9nkZx;vNQ#IhbAE zs_-{%BOOq=0=mDi(DfO7yp$*ooql#6(bN#3mZs9co}Ph*HM#g_$R9UOor2Y7!Z`eA z7H7@n9MFBR2zc8D2%b(SqS_h6XR#I;a!_X!-w5$If4S6SrVqW&1>E5*iFbdeljZA5d8>&)DiQA)MELf@lafADyHn)b& zyrB;zH|O%MH!S4k^`u~TrUDsNx()aA;_yPmR44$i1?n7_NWKBiiOsaCG$?_|#B@N8U@~L;jWQ4*MOD)c76L z`hHO3jk}nMyK5n$>i~Pp`zhFjlwwVSE(p(90Q^3Q5TkvT;hMUEv$8+^rL_*)*@JxknY+gZPQ-#{2j~ zL=>B051xqJhl{^`rT*edaQwImCLZ5QE}ZjaejT#HjL34@xOY3W?VUv?{Ka9xn>>&z z?;&cP->^9{f<&7u`Xd!Y%v?$*kv$#0`Oe-GlnZSy(d z`%R(vhA;k?Pno!aMLn;$8a8DxXYM}cYK9(2oYMl&oIregU>iu}=1}d?5Sq^s z=DZ$KfSzOXFnnPM)AU!C8GObOIr!f=HCTqA2NmYUu}W9TmR7C=3~@QVJWWk z>4xKS)o?)TA>s6lQRg8&)E#7C_tkW=e4Px@0bb>Or1|Ekq@aJ#dlnV{g<|;PX>%NKRPs z(!XtF3O^JA|I|=?@m3Z#9_=FHr#Yx~^)_f;h@?yrA7^Cy6P)+pJS}-Oi?(mkWKX&- zgAcx2I9`8;$f?A;=9U|xP~h_@&TdHt@n4g(bFn5z*IJWFe;cjc!z};DwusQ;;FChdPC*`k1=prJqRt^mNJ7X4)KhBx z%N&Fr?xo-7rJ*FPg{Cc^(cP&LBrfxVh2lx{tGH`Ek@$)#HLZv4DjR5D_K|#-00vHs$vNWccpXgD)Hnl=h4HG< zF6t4R%J7u#GN~0ja&`O{jgrbkr!%kNaK#+3Tm6Dswf2&^S?{5_-Uii|I1#zQ3RKuW z#u|1i0q0r_)>HtUdT%W}eJ%&9bd&JFP9#R6(;--^n>l$X5`^B#p^iZtvDQdN+cyue zKKl{xekyi+*76>>%zwIAlk@^siCZ1x?|2_dCKl%v2uPIp*S41uf34aD9qYr5B*n1oZ*Z{WP2XRH>He#xzPxnv$ z!&|pSbTBXhTg1Q6^VXSo#%41O4DW;1)@CLzhIlaiCBzv_Jc(=lJIR1#Bp&zVgTbX%C@6Cg+^4m&o-TxOz4sVo z3eE6|-6G)EQ9~=0ba1~h0Cz8j(qEq9Jf5~bZYZD4d5|QJPs{rtcFAm9cwQA>oV0?3 z`}09={#M`(<(Lic34va9AJ{6L2=Tdd@%y?Eu<;Q>jb0VnJ^4;H*dM($dhkzP2ISl| zLWkKAbomT3G}#wSUj^}_(9L|XULcRbKChslEgov#kI}6(pY(m50trVQ;Yj18H$Ml6 z)TU#&Vx~IO=^de)gA94ePn$^oh#7WNyr*B+-Z2~MzKa7clEhalkDiHl!I}FQW~);N zF73O(IF5ZG(KZ>x?mru_)JcZ?^g_0LSrn)(FJpVA8)1fAGqtG*CvyfPz=B27reg~+ zxVsi+7S-U-Rviq9c7v6(`{32=?9?W-Jtlbdb{VnUtink>okzosC5h5wCrI8Tg|)+Z@JH!BrZqgKPEOf$R*$mas7Q^D8685-zI(4sVhmBeN zVA4Ae&RU&>+FRbFa@#dL@YomSs}rC}G#~Olg~A=(3UcjQBBth2?2>lxG=+ zsT;S0)v|nW+@#ETq!%`c5khg7b`e$5nnoW@S0EK~qhwC|P0~HG3M{o(6ON}O?@sbQ z@-DlAS=}QJhB7zlr+zomzef@j`{vPCLb`AzGnQHCsY0$unDO3uB;d`N31|?yoqW2J z2U)>mtd!0c=>Mt#CY%Sr{q`<4u%PHGM09b zlTzZu=5-Afcqzb3S5AOuyTb6>hiTxlX(d$2m5`k|U&!TkrNF*gjXRFcC!#@nX^+nb zw7!r)e$UT=+6CqCxZx9+PVL8DpU05DiXmpb58$K9eHzgxf)X(o@kHS&xN$=e7W|6A zWr1Qm;fv2m^$8JpRpf`a#M1EBJ1wxy`UKk!{h?FzUGbM-IvLAW1GA7is&J8?^WxcF zc)#QpJ^i8uANLi(;`lT;C~%SHoV`x&$tcqCl|m3@5R8`I$#{FRPlkyc1_hhnbnjij zS+_WTzoEn(F9<^kZh#FD6Cd%opNCN%u%FLeaLSz+v z1NL8?Cq0);Tsd+CX3J#5)|rpMF0Pg8ei)qF_SFKG=s?ff+%V{lPVn+J!E}LE4mYTLWk2yn_A}8mOyZ3E}E-)a1b~ycy|F3>GBe>5@_MJkf@; zF(ewYpTA(E%R*tvNiWn3u)uiHEH?MQgU`td1ig{F^_$MRs z$u<#;mPo*f@eAb8?L@j%Ru=@XIRJ+SQ_0RF_+HN&>;H2BUhy4#edPt%o*$;;zFA~t zz7dEBFT(m0AE?GW5ppzoEBY0MQ^kc*t!b|HRjz&cU4!)@_cnFaJd8U4~Fu z^AyToNy3VxQC$9m$32rAMTH9*$fb8q7|s?`t2O9kAq*tZFpRx zHSlxSl6W?6yh4J*Y?fp)T@srxY3={WQr0IcQua2wI9m_)K;l ze3XkPPkshct@TeZPP7S|g&xqUe`9f9UonmO?GA^VBk6~>bSi%!ozAZb1dZqds1)fS zg+0@VkCZ1-ZK?+M_OHaMw-Ht+yr7@H7UOBF1?VF^0~flsLdH3B*s4Ddr~bDdhRdhW zKgLgB;AJs&_;njg&44aWET%6IX1f#Hxt~e6^)}jUTngyo+BTbA01Y>>|-d+4xHpmo&&6si9UH|WyYjUYW;|61-i>?48K3w#ktv{ zk7fT(!2C!azEe0&wz;oCiNeLaQxD?krkU&5ym>CPYV0KWF>#)HW<@jY272V3+zhs> zSC9JMb;A^01rT`~M|W7&(OL~b2%WZt#8X2|md$~)#%bs*5`r*m0W8&c4Y`k{(P4WV z*^!-wzn>h$-VbrGW@SC89!LTm@km%yFT@j-O-F_BC<3OPc=Br~_7qmbj7wsu`eF%a zG`!=ko1uq~raVV{=S4#F>mcyHCayUX#dwD|VyW~(6bmmS+HM4j>?PsL6an7$ufvp^ zeFFb8&Z3)Z`&r|&;b5zsO3t0Gf;rMgczahBx~;5(j!)fWx)nkDNv@yJwh1;U{>P~0 znWE9BH`MV#60y0mhHPGN5fmqR&y<@!czi^iGxe$=Aqkt{RjLr|lmK#h$$nbnZN$ry z562BbMWjejk5^opz__o{2DIA*H&kYU?v~?F8qx}B>!M&`j19TpIh9&8y=N38qiMf* zENCB>f@5=T(W=u|K*Kwk$}1niAbx%B{$KW}S8@o5mlF=J`pA~X9fltJrMNHfDd8EM z!UXUo^2=tz)WU^qm;F>`j(tCQniWf%Z|;WZz-wT_cMt^@^#M-q5>&4G2ZD!_=qsnq}9nLZVcwi2U0a1VQTnRlvieILv`nwfWwG5%~~ErW=oBe zxc%2CD`f^9F?G-?aSJu$%IPr$Sx!Zf9{IqNWalY|&Kov~ z!UOcgPljFI@t1Bq6aywd8N7FBI$4xF6_j60AzkGwVfI8f^>dpG2gfy`_QeTcwvS@x z1_8*rxfiO6pHg8Wj(*WbcuOG!Ifr!Mx|kGaqoX7q?j50x%l*tlBwF$KbVKaexSu`K z;f9SX)8Wy@0<TjDJH%*9-8h%Y89p{xa^X$A_^aq>cXE z{EaLNlp%_KVW66sLXvB5kW{Coc+IsK!#&#Q{rRS-c1MSf9SujPtB#O8nb)78jt=iH z;9A^{h0Zx|AZvIS-pss&E5(s2n9jiY=`2~jHH&0`EEXTr#wU*c_@G;xw_K+X#|E~; z;nVey9$E@9xAwt?us~dEn2Hx44dVW_vWRY#jPxW6PORx7T9dqB|Ccd({agtyZY+m2 zqtltgr^dnBfMqmhYZ8lLA1r;@K|4d6u>FV0@^&&wVSZ1{t_;=e@o8&C?)Gw)R0u440@x)j_Rqs=hp6MrJmnb0sB52 z|5LX{W1n%x*p*9TK1Sn;g`#jUPzv+F3De>XDPO4@-sBf&kLc9WqsD?Llg*F$ug}6v zF_!#y_a*L{whLsLDX<6L6V>&*A(G=`{*YA$%SWH6@NXy3-m;TebIQ@8C2aB&A;AEe%S90%qW1?-u&`*mZLDXehi39ieM357J9fl+=4l zLD_HsZpiHdC|AdU|9s&2%0V*Ccs5wq&qbBXU1(^3jVl!!&HczZjt+l4Abf@cyL+xJ znm5UUYT_J7+O-Uudo#f{emZAvXe6#wG^bHfrKI`uOwO|oGt3A-jG@*?FvBqeycRFP zJ$YO3de8x~=usvyl0Of%7F=ff&{;A!t(^_~T8Inwtfl>B0nBsrek#JbhdZp|(Pq6W zJ9N(rvLVgtDkML9og4Dd-(8XkET0GHeI$dl70#Kxut zgon$J_*LTJG!1&aGmvSjucE*FlsMuYS#;BcICftC%S?+p33*i}SQ0Epm7n;-JHap* z?A$><-c{uMJog7Oe_zK%2^~0keGRO;T?WQld+A^-AIAN)Lyy2@EIS$x3MRK`?zszu z|Hd$SPR2;yNjYdzF2#e{A!L!)MYAg3cQmfqgy%R-8<%%nAPrvPC_H61txHBY$IW8D z8wm3}Y#6E<+roISc0fM`bFkN`M}Bb$s&W`_2=MZa_ zcnN;&Z$S@-0NkRG0S{)@O=|M_cymn~>~A{-f8@_ow^y@y=K`Oj<=OdY7P$gKw_1SL z(-ErOc7WfJ0@D=~mc_Kcb+s16-54T^VA8{A8fX38Z;*b0dT2;jHH&)PNo>=-FupAj zZqX6;Qg`X@FZw6PO>E)N75R;yPv1? z)2fMSQ8if4rzGaAE=kE+&VG5BFzIDG)2gBg;LhREX^nH~{Gl8ev%Z5zdCK7F(t;;; zJs?6!HQW_<=fJJBKpgoYiZ1`OVONL%9^JnjGW0i-M3Vuk=#xwBD)yKiIN?eQM@_*& zfztVvQn*}rKWtF#pe>ukP+4{^?}+3%TwT5y`BqKddyG-{%4v{3yb3waM)-L5Trw;e zjc@huf#%kA@H#jdJ1`JFyJWx<{RGrgPsXX1b0)pRD*VHKz9z`i zGrN9J1<%X)*jF9`8)w4(VI{mg7KL7VO)ymGM=578AC7!a?6XU@3SUmu%Y(w=-0rq3s!4!z@AGdV6$U zx`*7!b3&eX6s)A>aK~X0n{X>0BCDU$#@?m)Db2e(b}__^u;J-&JaSZy|ebIN<6dDai+{d-OP=Z=)5X;BlF*&k&;|8cwOknL)yP%oV z55BRP#Guj$JVN$jZhb9n=*)wqYh=)87lnX*G2rFFfgZ&atl(zT-uLP-vBeKF@>kHy dksRP2or$Ev37h=i;fMbw=g!Y7Ak>iWE(xLNwBFr@8;RFB&LPM1>SG=1xT@B@LRSK_ks2Qr)T0psxS9 zFLO#FglJF_O(Kb;9`F0*dB45KT6?c`to{91$8S&LUnJP6$;bD9W||BipClh2U$rP7 zU-*FpQ-##M^i_d6b{=(ralK|(78?ZJs{i1t=~M7?MG#b|_tAQPOEd_KgVm>^9_LfwBwPm!kqn zDnei+>xHsGUm;)~4~ylynH_7|;ni^&2$<^tQcuEhURgQ$Z$OH*z8sJC$96#1qmB63 zQx}E(+sKz|bLp$CD~R)fjdVrFO=x=ZFCO|mNEMua5Uo!E*rRv`xwB@I4iOg^@Hq_% zUA0WN_;RdKiXq-Z;*d5soUHkk3Bpft(!*Q%fxvugCIVq4EVfcp}1I>b3(@g4ldB9 zA8RN9&1Qf&Fo$;5s-JD#wnC@qf*DZK=#%+dJf1C zI=h&f*v>?S6RHq)e>-itz|SkSZNNJ%hEN*+4DmVuiJD?0Ts7&?+bgW*LjibKm4Y5d zD;VRq=kauc3JE^F3@QiJ$hJl~SiVvhL(2l$q0*!DfO9%+{;?ik%d}BBk64`3`WiKM z>*5Ep2&S%ilAW(}u)4E}I$7vIVNMU-;a7nZ0pgrhW9n38I0-9UwMe5~DXVIGAD0^< zBh%s!0zrrAx>f<+R`F7D)o}qVP?W~qj3ibEiF0a$RPph}IGoLGG;LR%2T3+Fq4~Q! z@5VAUY&o9@ugnUFyJi+z%*ci)ffJzqE|uP0x(oPzPQlP=ADdq1j^zZliFogi<_pRgCNF9mDS1dtoZ!y22jOhejMc6gl~ z@$fsvt(oPIrg}9{#~p?-zFD9vKOb)E2$7rez7XiVkY}ZEsF0^t!!ZnuzeXm~fR> zSK9-)=<736^Tva;@TxpmJUYl#)U%>Vkv91KpcvC{6h`O1b-~PyO2lVd4XHFMp_}{< zL%vD`vEP!9HCn~6X{$N#n+Cxxu|~2fQ;Ij(8V$h{zRa>WH%W$r2n@X%gN-jmak0^K z+}HLuYeph5@j^J}3JqfJkydiVYZ4~c3veE<+=k7L(NwiP6Ox_V$Vn}XvGmgX+-K}HqJCikG|RRz?U z8_eoS>>%ECParCd!tajF5bF4o`DpEd+a_D!WAZ6#As$aO(iXw+x&TJiF%g!ieJ1WQ zr{JVW4=FK~V{?X5nT{`k>`AwLY+Vq9;Ykcg>gMA1A5Un*!EjXC)<^p%6==twCVKfr zC06`qsNuL#j~dyam?5hZ$2<5v-|7NUuGKdOb^6xd1vf% zx&kG)Wtn={{b;!nh|fYUHuCP0G;R!t?N@_I84-@I{7=*poJL-b731hm9XjDsZ~W(z z1n2c@A66(QhZQRff{g7>>wttcW#nu%_P@FNeCX z{UE7T3wzC^c+EdEsm$*Gu+!8F+9N_t?k|^v()vequ}BzRUgLsyjPBuGnv0hzxcKS( z8NA;VjR_AP5s$J8^2Xp1fqf~E<(b2t=w5@DB4%?AzTOYjzvPHiO(Ao_CW%}c*+AFX z2$1q?Q)JDg5mu-_!!54_c@8#iusX9I&#xLHUv4L&==Jm1`PKlF>Mcp)a6Vdnk;f-x z&D?jN({R_*N>B)RhjhOy&Fsnp#mGVo3+-g(X3ZoaksHvuG>d&+GY2GAe4%p1W-wFo z5g9R*LJzxm5WB|UbGhvhCu4*zFLJ5zIc0ixTM7AcYynR&xR2y?K4YWSCy;rU_`ojr z5lQpq8E4WvXHNzigPEd=)?IeY8{gR zet}AA87RPW*`bB^&-6gMQZrfIZ4O^!`ABk~5mkiKrd6B#iJa>>oatl+q8H+zc7qii z@0pIgK?bL3z6a6xUYZUnkgm9zs-EPE2cjfex97Jb9$fNnfx(SsNRclwwQ8FS1XvlL!u7r2RT0*!uDa zEnRh$)J5Auyk{qjsKC1UHA5z^xQ@;&f;+$Q`1P2&&rhe5 zuI9wI&J-`kJT{fW6i8QU!7ctXcycGRLEfnr8!p_T_V?vrY3dPnR3?SJZl1(s{j1KMr~6LT%#s9OfPxNkORV$%#13><;YT>=~z-)nIBV*+btbCSGV zehK5Nd+FFm1qjKi#Fr5l=(6FTkR_o&zu26C4MGy&8csy@+l9YQsWNr6o z!LKd5!D~$w^(nYcBpP+#Uy`k~;2h-(( zt8lEK06q;TcQcC*h^ynBG`AXvNh)Pz z2WtarnbtZ2-}l?VkAkPrd#-{y6pCWnw-2PsITajVcrd~h*-%uV4TnWH(A)arkd@m` zlk9!SnjKMiGdY^g?LQ5yn>hr0QGh7zL{fT9i41Dyq3BF|nm1)kdZuQKQ!6ayo{3cRX6pK1TIedMV2 z8<4O)kIbS1jM8$(u{>RPY0XEhu2$pQ4-HIc@&gz@7(v>CRmtNe`^XKW7i70~9tNE9 zppz1YloK^Zi!N)yUYjLYXH!ibZE|s-{5FYM1v2=23oocDPQL zZvWEEgyjYke<>}Ty_pLYrz6-rpY=5GA_1xQsir*rr5uTG4pepV&A+(|c<8MQIEZHv z!nuMtM1Yyx|dn*{d?(TYNhNrdQCH zJr=k{>ni^Ajv!uI3$Vm@7CNUqh4|a4v{+LbY6hJc*ApcuvTOhYGk%eEi&|lpWiHuy zPZ3_G?Z-Kvf}gTW9}OgPy_ z^BSAU<7dIxALC1n`GeqDlpq|qu8z+>hk&$HI=Pz|MbeuEaP6v=+IQ-fIOX;e`#EuR zRDGO1oIHzksiecw31ytL;!=a9=3sr9gJ-MG00UPp~K_KXyY{kd0*Su$s3}qrT%$xAXJ$%@mPfOXH!1h&eiAC zHF=}JClwI0N@sk%yHG@KkYw2Wrnjx`!3It}-R;s#FPhu~?yPWj#hWC^w|GO6l{!gm zuODrR=Y#soLR9>L4O)f1WbN95MQ&dO?P>Zt%pQ*mITevatxEaIB0ydZj< zEK;i)1m61Vpxk-`#BW~7tgg65)R$Z)2cy%FX^sTD;+NRAQ6DOAL=xlJ^HkEhj#O`G zA%FH3V2)V@D6ol4;!;VDr=~S@Y_?|HuV~||sV>&oV?HrRtOad>ObAlzC2xJYKwg-i z=lgAdth|*2T}x+y&&49Nd{Yi=?`km1E+xXFH9B6402=jUixu+232zihOpl4P^-C~5m_2b6aB=QOf_S?{BS;v$seaR5l`3+ zR-b6)Vj&O=-3apTcj(qvw)lFvKDg^JPLYgTePCfXDid@E} zUu9t+Za#Zhvl3oO7Seg*`P_=XTspKV2HiSaaASWI+_dheytYDE5~snlQF#CT diff --git a/tests/v2/fixture/test_format_compatibility/array_13/.zgroup b/tests/v2/fixture/test_format_compatibility/array_13/.zgroup deleted file mode 100644 index 3b7daf227c..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_13/.zgroup +++ /dev/null @@ -1,3 +0,0 @@ -{ - "zarr_format": 2 -} \ No newline at end of file diff --git a/tests/v2/fixture/test_format_compatibility/array_13/compressor_0/.zarray b/tests/v2/fixture/test_format_compatibility/array_13/compressor_0/.zarray deleted file mode 100644 index 7170ee6829..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_13/compressor_0/.zarray +++ /dev/null @@ -1,14 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": null, - "dtype": "(nw5voIyQ6eKnDUnFON~nw!BBfMzl&tKs_c+(yd+%#s z`?~i2-T%LTec$&z=RD8zIpRVREIWec(6;Aj!sDgGn_!HAY{q0^kRUvU8<_1mp)mTy7C@!F+?t$Dn^A!XazN`{x7T_=9)wZfXU z=8+pTMAYda7VKu|!HW*L?;K`_;9{4C)>G>UY@6PdLnbf~R<5deyZh3}e~kZrjytOh z(hpse6by~Uzc7@ewK0Nt{_%&ATd~(D99v+T@#^6P8;<@1-rqGELp_+sk z>sGjUjDG*^4gk8sk#ohlzqY0Lx%|0&N$5LY1IL@L&}L!Lpu2x z-lH$Pa>vkwE0mo+a2YnA4cz}M-3#0FLbRNDt0~_w4ii%me*csuMR@K}&|4I3-9r;EjKje(D}5oI^$<%gr>I zB~%<2UR;6<9J?}^u9G$cJ~#ZUvO{SI`%Ws=+={M%|0d-f^*tvch*QeP{>3~h71`al zRvp3`(@EQA%>j^GjM^O|PHBBv7A$ay| zyS*R@`}vME{F5VswEE@*!z)_(Naxy5r`buc+bb>QGwT@Gy$!Wocrg!2WND6!Dk3y~ zm$m4V9)o@LLL2LoE!dEkT_f?80^6&f1lYgqgnVJ$(o>O3SnMeP6$k6!lOwDgE{q_+cg;w2S7_Px+THY$YH=AH~@BF=EH~XOPafs5e*Ay(-dff=FAmb5@Mfn}1 zVOU)3lUI5-giG;PV+vo)0$eRnFBzJ}wYNrpb&uqN?a`p@1UCxyz3(yl#7Tg{fv3NA zi_*x8OYf`>**^+yZ_ju9OAW!5As731$@GNV2VymD^Yx?XvAYRb_j}>U^w~3%TSUB^ z;{P_NWf9VZ_$0J7lfhG}Wo}Yo7L>j%t(=yv1Rc&C+N78Utci&TmlEnl$J~ae<>F+_ z(pXT;_LzmfqrrloO}cR0C8;MWoe0S;fA~fliOBQ0=P0|*EOxD^#-tbZL$pTa_pPB( z+}BYl?;F~IWfSYAA*hP6z-p$>_VhX(RHSsPP>W9S5cFBCnr> z6heH?ZQq9E?ecsgtuqt+U2w{4z}vHU4Q))1^5#Gl>QD8b8xbU6Qo7G-PuL(bT`-FL z?UxMZWnZ7A`3>O}rZw}4Yb_wM^Aq>_$4z7nJaW|bIt6rH8_oX8uV7&5iNgnb>rwj_ zhqLaBQ9KiWpiJDS3DTG;?UvSM_~<~H$M&3fQa7+d@4J6kZPmP5IbI|lq8S9G*18u8Q4qeLF+ecJvk#8P%=2VA{wC9%71 z8ejTtz98|GVTV#i?p@C%tj#J9&Qzu&Ec_Rj|Iu(7EsZ($+8QB*&bvUb!=WmWc$>hNDxtGN_bWuk7WbXhL3aQFFm(vh>mP*o$ucTr2 z&Ek?p>N!6$xW{*re;#(M4ENqrjsr8UXNj86CsDDS{($n}Bv=IQvYmMK74IgGyq9^@ zh9*twmj7K{!eRHs;djk6@@BHdis_bYt2x7VhUtg_7H~rezx-*A;N#O>=XGnmhk4R%Vpt#4V?4HmbS0kLXq?D{AD6& z2;a+TmrFYvpv3Z$WB0>RB(r_Lt#ExA?`xZK%!m1d1ILRUqe-*amK@$OZ88G~6vt%` zNi2a`Drc8*XBGArtoCT`ScF~@r;P>OA_!kOZ_*i^2JWgwDKfchaArqPM%2(C^6|=s zWvOi89hFy~b9$FC;j-uP^{7+d)J3K*ciRHw5}Wo2P@VBe;+P@ zg!ARMx%&szO#{>|<$=TAOg{ZS)T~m8=5rwbo<@0WS zgdgr5HmA=}V5coRe3Ban2CiQ&Vt3cTI```5j0b%f_2s<>6PX0R{q~+NI7CKrkZ{wx znhMbCVwosTn1ySJ!^fY}PQi|{lZl%TvVk>bUZGf^0R^uAu8C@yL6_~C_Pit2IJhHP zJz#MjDPIjw4K21{w$}?ywUKpv#~fFC>Cgf)Df#~pQ=5SI275T)WsL&Itn?pyheo_y zB)yX0lL(SrOwHyt6;Lb8Bf0Hm4=^Sr{muK?3Iro-&S%C8C`a7!-!l(-!jH8FQ^HDQ z6g2T?JXBDGycvp|OaQvd&?y0b8eo6nVHSRyO2INI>% zj(-dAQdug_GMI?lc8SgvHNSz0sPyXzEIAmaldSK#qa3|$dj;8(SAoePSNrAnCeRn7 zD8AxaK`#qiLo@4TFsETZ>*3b`cY8u!M3CC>aKkOe8~zKx$Py>TG(p0LF_mN|t2HRS zWaIa5AOnuOhV~V+lOf{K?#~xLZI@Sinx}PPcNx5`?@w8&$ixIU+psVDX0eq%F(9=k zAJf*>I|Y5n7~;Ir?K545S*%I|nu1L7Z&bc#EdR=f++*SWwHf_*tSB~@xxEr&cG-0u zy3>i7x`WILrDSjpzw&+IUY>`Pjde|5GEO6HsIzge3Fol?8&x|>OG z`PcI=!oKu`MZeeGV)@l?A}rR1$!i4}0QZvJ7@QY%JkcFIN=jKFVUJFZoOQVg=Mh;~z?m5(64e)lhB z5e)bmEAIBrqvt}RZJ1Lz{I|DT(Zp>OguH~D?6rDurE7dy-E3<##qjuF<^pW3u%;;+PD1VNr7Z48v$#FvN=@ytdT?46 z;c1qnA=rMQH(mOVgg;Gnj^=%s1$k#HhIZ9YDCurTL$FE0{dI472L+MJ2u;_EY6o#cYTNj?)=psMZ1}0KNkQp>uh|DOMuGVFz@b~;n~|AL=E05i z2yE4V!%!#EhPi2HX@*YL;^CZqbp~u?Cm2@QCV9C92k{%rps zheYakMZxKI-)UOV=YVz1ra>R3It%xom1ux}K-ay^ij4A^jBb<~3T}y5a+~@NqgldB zF`daFycln9W4}Zrubydnp{aEly2Z$YM(L~Ar01sbmbi&5CEv0}=h9GzJvVa5cN0$T zJ@qU|a~Ljf7g{iB4Z)#Rn-f~6d+=TC=gd9lXMyFs^l^joQmE2ZQdUW$_Knn=^{q5( z7!{EwLw~djK4iUXc4r*GTvFm$2bWF=4dXXr+EWj5w3g0d8bkUTNRd__X z;Jtu11L);GYf?Ze?c=V4lq@`N!K2s2F^`-(QS8)-3H(qk{bHxsH2RLRXzTo%!oKS^ zouTIk;NvcrU3L)@;Qe*|N{ZMTe0PZAZ`B*Yndhh4Hz_S};i;q5v(zDMWhGy}6XXLw z6tr)JCoJJ}SC4PMmS-_}Wrh@&JO->rnpIbFm(aReZ!;-=5dQ3wxk(>Bj0GocFKpFr zqSnx1%Z=*{ge9jD%bSmvKuhxFs;|Q)-fQ@%l4X~ORnA}Kc%oWC^|!Ddk!}IiGPGLk z()-ab(TREYpI*2re+`mK8o_C!g}m3U2T2=u_6=#(!m$SNox6pzF?f!BU)1gye6str zy`&b&CBd&ap%dHWAA6#a%E`JBx z{~WYzx;KXLSI1;8cC3RS^GEK`p(V6z6XE1z{0mEebYHTXwL`vHyF%NcKVZ?5x?ko+ zIc7MX&|2qi$K+EqPa{*t@KM}I+{EQ^3}g=PGWkdW1BtL*K1$vA?p2>b#vWREiT8{j z79I`5RiEc@U2hxVKBKpJwBjU$`tD~hl8(chPTV^$URXqRxSk^0I}e-!7kF7my0EZG zU(YU%h*3to8%OBd!B|J<)pk`1Y?$yUmu=63nBGILZkhjq@aW9>tZP*LZ+-OFA$kfJ z-(d(*c!yBo_ivk4S2fZse&f(GY{i1UwRqxxWZ=s#CXp)YQOScT-262eh(^a_l|_3o zORUkD`{*bfmgG;Y*)qb0BnF+>R0jp7VHaVqKJ z3UXWAi!NN)f-UaK79_Nz-f*JHDBol$WixoxOHz1zMk0K{Kk$}{&HOG3Fe3WD9lmSAy_$y z4ks3rUuzE_uF8c91uUS!P~c^`i2>L%x72mAa0S!1iod=-VA2aI^A-O10b05q}J7i2#H)-R`g91aHCr|Gv96&w=7FN zWvKV}Qw-uFu9BeZqm|gPdsI=S!Hpfjq<6$Jip}VB^nrkGG ziZJH=eRd4*OzhMaIkO0V-+M-0WA1{wioR6!8EPG$&R5O7&cABESc zAtLE%MELb+Elc;`B2@qVnm=q@g5y5K%XeRofUe?Gr+=#>;3H7{1Z`U|$>8qgT9G^? zR9(I8KjVfEb&qQW*UVw?&~d%P#CH7lU{gwsyaY_bQ@IX-R3E5lxv0f+0Oe#~rKTPG zhRhP}5OQD!664aL@P0SE;`Z(A4H$#JhZQ@$Ir0R zhIVO}&M#oyvby|8-7LNg(ofP0A*0#J;6e z*fxs{qTPDjofer5lr~uu{;59&?!s4A_9<`St#{3VPoFKK&wd3KjhoNmLoA)eHo`1e zHQc)Uv@{9ZmwXyEx#lt21MbQgmch5za-}BZ9^~gU()`Rtf#lGEyHE5d;LoHpv-#3IHjno%RjB5bPD5DcpSIP;X%Ku(Z{B~o6?XpfDty`A1Xrpb za~6eEBZJrh=}2{||K#N|@%XPN_TAjk#Fn)MY3ZwPwe)L1MAl5|wEF^{kUT2&Tz3tB z_NvS4(=8#ZSxfsTp>`Dd_V`n}{5;&>G3tK$J{ff)Nf&H!5#Dbfml9eog=g_1>3QT{ zFm-tmM-b})ape!B?2!T3_Tl+hopvYo|McHK5Kl)i@{v2-*ffAkF$aD+nNOnpV#X^4 zViTNf(!9BiwHj)=HD+UmDOu$3HLa^o2UEaL;d}r z>$8k=z#H|Jq5Xa%{QDuHDAd%7w0>`T)=Fz3JfBDH{hPVzFJ>Xu4(SWuRW#LNPXjErRKb?VUOwY&i>?T0z*jJkA zD--aYMSsl6i2~=vtX{KkEr5=K%#&=B2rL%A_&oRQBB;?k6WDcp2<=L|wAVj2q5CHt zR;6oYc(&*3E&fg-KC#@jILU<0Vdz_&tc}+F{YD3M07M z_&0{;-UN!Y)B3XHqyu}qvGeh(E111zU{?Kh5-W0EvS{D!!5#5h(bWTFVASCVm>(x# zTC$M;76%!Myamjy=*oe)ySv zvl>P=ULVq>)=9Kxb%;KnSc|1(4sDW3GQ1ae?hv|Gg|BYxnl5)+hOs%n%pmMAfD*O|=7aXAYrITrV`eeJ@F?J`60)Aak*L08=b( z=Xm^`Mez@>-EytcfwGwA)LGvJTFpO`yPphU`YsQ$6wf#alqUaRw;6(%l9$ytsQ#4V zo{U+(0~9#$wyWf(7X@h8AA0+KqLB{^vJo^iUW1~H1un^fNfb!@I&1x73YI0z$l+)Y^uNzPh|HTfO8oWDhB^=LdoN@x{A>-MUTo<09cu?Vo_AYC zGhJ}}$3?jt9~#imaXOszaSe9WmI!e9%|HRIe|w_W3fieKoRz09#q7mI^2pw4ObM0z zKyvH^|0spg{hGg#>j<&iKw%Q=ADPR3J+cOhd-(rw8~+7Qwy5OPD)?=fIBd>UfJHieQ!pY$TV2l4JBMQ;#a!ZA7X9~a}6 zaF1!)6vbi+MR7Oj$?ZVgsJ_9RTswnH)rHpumY3jLyTb&1+Z65~+bMc_ltKRbEPDpm z0*JOW&W`@efKa2RfQM>r2v_nRi8K^K!SjbJ*(?3XDSq#rCOsKBuX;(T1r0)*(mT_R z=mC`A99!*QnSzCwdtB>wV{pO!3Qf0a3CPi9sl44g18(YHiPL8NsDXUfQ}&NR#|MkF zz2hX{^@}oiqt=AVEztzmcU@S~Kbv2(rv;A5w+&Cd*@99}YVwvdE zO6{aT$p_cKP$b*^TV|xOaa4Sa9?hLUpExiEt*U zSO|OV>sh}ioxZI!1$9I}ZgC)1;^wlAV#pkualvDm>YV8lqB*AFY4I4!}{ zk4d>(T2B+EVDL^|rhYaF*7cgFZm(70F8}p`r=q=7-TD1~am_TmRXy68wsQdQC_JDS zUv0rVRujyHzTLpGH6X1YypGk!Be@i-d%({^%knG@1&_&;t1%v*hBx{7bupr}^2VZ@ zlV>$XFs$Se6CYI<#21&XUSul6Dx*867*^|%FGgA|l5ZH!KP>glT&MxzfHP`rON}_N zdu_?;Oa_c!mFKuQKa2UUe(O{2EF*eSgX zp#>}W;Nx0-en}@@{PAT}_kJU?muHc$e<%Q^sOmW8eFJD2t={F&K7sEX$(l*ii%@g< z)|rA73Vh^>bUfzJwJU@vV=oI6wrM;P zoX;uQN%eg+UM-&QUqRkr_87K}4v;>;q;O`g7+xivS9tZH38l^*NOdUc!jk5uyrU`` zXs_`&xx=g;I;Z|pYQ>weZoiWrUHl|mI5CU6GYWybor}HPcLW3$F!^xw8!WjQZFNqe z40a_C>Q+=ufzV%RE~$`mYTm2W-g~AOoQ99fratIKo(0d>N6HD9o+7o4?d=MDpmpeF zdOHOE-uuErwfm8&+{x|m-Y)oL*%eV^H;;D`7RnektI@Jy^sklkH1gX{E;CJbgN7dt zkwnY!?TM$O&U}rKx8Go;54E8;0Q z_xsm#!L$r`;TT#Yb+!?;Y8A__@zD|F^XY943T%P1Z=7sH;}+@^$^< zn*H^Mgwj{<)dez4;=fK8?uvvVeD-HwZ`u3+a$V(~yjoB78Pqz)Hf*|Z*WOhP7XuPl z-mNI4dp8U{x_L9wAD59g?T2L2<^X;P%G6Y$n+GetXROy78o~POWc|{k7BJCZP-^$9 zg_k?4dG}yF`V;f6Ni+PxJ@*tF>^eFyIaP32^uYvdx3=H5>v9J!^fNJJjOXHwN&)kM z52H|FcUse9wHZs}&U{ScOvb3}_m?v!TT!w$?0QshIzBsFB9~OqjHg2#Y4c9>W6XB9 z(a%vMIP&UKThG2y{Bq5OKUIwkr?Osu(OwvY%bvGsb*S(7n!uswVz0WeYCMEi>AxWy zNKp5>vbBsXU9&AGgBze!yomSNk4)5ZF;G~i=GrYwGmU=tTQG#vl=o>X3Ej2tsU4Y_ zz+R?I`|go$l$yK{Ge0;0rnKWu%AOH{N5wQT7-3b3DZ<;tOxGZ6W;H1F)M3HUA8;CP#>2!E~W zl@+Qlqd;H$Dfw4naG|}ot!;S(Ek4GDacT5o65H~I_9CJzs*dblBBMfl+*Vo93OXM$ zy_1;G2(O-d>YcR>0`+|Rh={u%@ORyL;j_1hP`c9GXI^ z4_o2EhC9#EPxJ8B@X5h=sYyI=y4Nr^tPYu8&qYo$x8XtYX&R&cDPWn)C@5+g0%GGc zejmpw;4Ij75N}iidu?Iv{l^q+*XdQ>`GW)};A@VZ{1|vse73}N=;SwE98=fh=>-M* zu&qNEhM;tG;ReI^UU>AjzUtA{5p1dZsQ-Cm4y0UL^@={lQ|st&VPbMQ@|<62_^wcZ z9{#6J{_9UftD9_o+B|)@ljnjy`EwOy&aFJw&B;V!1#7qbr83-e!lqd5b{_;w<%tcN zroiT%`s?;N{rH?yH*WGN2}urZ44tR5VA~OWYcncmq5N-HY44eFWa8OPb0weJX9V)? z<8IVLN2JXL9-M=;gZacXg;tQd>^{|Ye-cjL-Y>A;mW0MK&DPH(3gL#palg11t1yvc zt3`M}4vojU?YY*cfNl8S$81tP9-|y%iuJa^eg7o(m@RfdQq}Reh)09ibmZxi%27Ik zO_369#=Q=>5hfAkQZx^%)PiXOO46e;Gz4QeTMlpS z7JO(ID`ea@iwlp^n_e1^AP60G3Xi1Vr3-oD1CH}}>HV+M?W|a z(IjGW*7X{eY-;WveCpT$Hwl(jy{rYD2~fNB!LEWtN66|A^AYHt0Yf$3UQ?@NxUqXc zW;BunUH7X<2b!x<#&3Clef$cUl@GQw;at{nx{B7g6udKUdptGpNaNGRCgk3sV2a@hm?7f>Pbh%t04^U{g!_qQ68x z)XRUNlek<4{%<9Uex9gD#=O7JotVjJ>G0F^Q{e(!a^r~)Pu+sMH`u?bkk+8&p=^Jz z;RuAuJG=4JtfJ|_1FeCAbzJWJBlz!gKML4289dw`k0O3kpCIqk1 zy;NSrI>O`I8yPF`sdDwC4RwC~Z8a}_eQh2UJllKv%QsL`MPTsrN;*>R)z`R+&x41v zzwfeb2rBo``W_Cihs1`Nc{#prY`+}psgzX+`+sO37L=KWT)UsDTlYG_vEN8PNL&23Vt|~(Ryco7Ji*GIN`z605jJ*#5qicz+2Su zYsuOebd}nLG<9!*ckR&%$$cB3`bxhvJ-!*-xh(j(&J}~37_;$p>fBqS{oe|2{V3d# z6*7nm8G(MIK}TV|QLN@oRd`ADH`jJ*Yn?kz2Epo-Y(>f}Di5-C9nvU=J+mgsKbsPf z$z|T)BOV3d2eHX3t>I{0L=k4smX~fN}Zd>gF@F|y0UD!&+kZdWVu!>PwB7a2)<^5VTxfD&UXk8Ay7uh>b^2ToCo3A@o|F z*kq&5@3XqX22XCyftUcF+g`U>Fl%j8On61W=Gy&?_F`=~%6s#}iH8$Zeu3NL-g+6D zHSAG{z152f-r{E_4c35O{Gr=)`5X=yiEd9h+5!D-6|IX$*5ToH!iZ4?2@0cpzVFzW zMZ)W${~r8Z#(k+4`zpUS0E1Pa(5=fw@W5{C2PI<;R0PxB4Sep0-Y0I`gPHm;>FJZ| z)`Wi4`#JM#+sHg_-^-Cj_mu#qyCbrLUsH1(`X;uBV^c8W=*e`aa{!Ve?=Agp?FX4+ zM==Th1r*QtCq{Fo2>)Ht;(9RNj`6r};{Ar)cV=gDI9l)^a{FuNf|j;PJ;DL-Tt#QGdM z9ke*lcxwJ!Kp&sIr)F!GAq_{)TIM z0_}Kt@jqwlBV-&DJ7t!=z7Dy!<}GD7=Wr^eJ!ik-C}yci>S9#aFSR{KZzkJN!+B*(l1xfuM< zn(Rn7AHnsV_OWsCgSg3hY#aI10_d9MvL2$&bw`!druG%>FX~ySD;9 z1`V#G#iZPeIeKayP~+M%LFHWc{ki3E703`qc6gBNKLj^b%%r3Iy71c|e@^E0RGe~D zUn$w=i$|^(9JrXW0{p+~_g@AI$}_6wKG@NXPdLUOM`*6#m}t7GRVIS%>vZ9Yxk3DV zrDwHbFA>jwzL2F>umpts_slzQkKamDR zL$lVg9j4c#RCTU+VY<`jBY|~`cvzRA?c-=8jw-qOXkQx!6(whOksS+Q@Rbr4L*<#O z7B%^#AGSd0F0(0<8%xj=sxQKLybE+vpXo`*rQ>4uo|iSzOYo9w=b*u^QD7s8ouO@+ z2M(+4EYE4?(7N?Lo7RU3ygpW(ASyVHeVw`;*3|wMVAp#5fC?!`B^GX+i+4&ccqwl25A2B;p)7-4On1v?janI8S{tBQ+GDv&e+2kVVeVj^1eeClw0t!$;J-0!WM|gSN=42 zj*Q*9m*sz77=h5`T8W3a zpPrk;tvp7zKsI zF|^H?mOzm2dVF7D6eevyOMLX*!cvK7)i3H4{AXKyB#*Zdt{i*8wPz_EKJa@vO_nu- z?bb7m?|Kekah@q>u&ou^Zeli{tZwYd zF?{1@>-%408&(C>)o<+_L4h6R73Ng#+0@f1Pug@Cl81j??bIHFA6nMo??jg&;)IK= zA@?$#pZ&%1W~vkCcAYa5ikO456qR!K={0;LnO@Y=K*4+SY^sM-2Qgva?&qcYH1a77 zza{U9&0yyZDNP-{SrBF%YNPr7lbX*32od!t80VdNmf4h1UVH3ajm&sCFJ9}C@m`0JJu{@px+(NXyaKoRY2-DMm(RU< zG=l0VAsz6$8T`-DvszHO^J8ldgUG&Q;4mAruZ?TMRNLwHnj*AXI#4raIrnfc! ztu8`kySL*f*CnVAx}`>IxQt+}tS2Y30!pLf$0FsL@bpGUq@6j9e5Xn4aPZb3eEe^F zdfkC~*fl)5?X8s)_(-XLxW49rg7Y_Oee#F#g5bi@O7>Yi=Mn7LDcOe1G1|QSyQ%#H zzgc+STExo4`KN|f3t;E><6eM%Aw-|kWoe@wgCkr@XXInXVdfH_D^1!IFv#!Vwj}#O z`M|-3jOtim?Q02r{&NzN?2jh%#gNfF;kx&`BXcO8P$vB}uLp~jJ!RRRHv%KyZsWn6 zO*lmmoj(0N8vg0`FKEzE`EjBVU#KH>&aKmubl=v410mT_Ci)aOgSOjw*y}J}@qSf3 zTRj+xixNwX7O*X!=C=dyGQK!fSmTk=4=hu=llr1(;dP15$+C|fh zFH-i0KewHSzu6s+w!Fj9_{?bWf4)-?*na-VFNY>%xkgiUbZ(nGkBt9T)MyW^_~rG8 z3a!HRG_&Rsi4k;?etPyua5o;_xo2q`Z#~AexQL~e)5}Ml-51O%SA};N+-%#pVGupSQEwYj3tX1Uy8k`}4ay;H}f}-e=X++_C1D zYNdT6h}WMi-&&+{Cq=gjh9+dlRbtZ+Fki<@rc(#^mC-RgQ<`LGWI)CgQUmkOT?-(5pKsx~D4qPvHz|GF z1UCTuY$P+xlkxNBbEcYpGO%6}(O_B}M(MZrq-d%6?X^#%ye|I+@Um?WO9baU{5a6X zo4zs&?^R@)zXVbDf7tuq^xm#P_~PG|skj2-ACjaOC5Rwm+7%joEd^c{efoL*dkfsY zqu#-CY7OSNYK_zf30QFn@4hCa;#}O-Mo%{y!brB6*XzbYG;hjOyFO75F2%pBbWKRu zu(Pd1n464i5&C-Z!5jEpL0!luZ2%W`kkt6|7U6VaT45}e<7|BV?QkfJL2q!ZVG!>m z*xwl(v-wRgzdiV~Hs>!goLo_{8~2z785aWk&bI@b~#8%wBlMCi-6xOtFi`gov$l}Ub=c?*9XZ!!o;V&bV=S@4POD*Lie2HvH$ zI{fBX0qkYh?rQx$2dRa-Ur4E@!<`PT(sikFV35ugb)B!nx7_i~8Q1ziXk}4DjFX68 zm?o011yg;ptWAsP$ZmK%FS`3{+!mD^`Ri%v*@#o;N;68M$;c#m8(v3J`4w&>-5l*G zFcJQ$)+jR#FE(O%617KBtb6gcKizv^{9}?4XSN7dUt?GK^9Zo*Vg4Q~okAF#QJPS0 zSVF?N?uPqfV?Zh8y-YbVhC0qam(<3oI`MdGS;%Mw80=kl(+zKgyyCgwm`oDn?@Hr; z%s}1K;*z1Oa9o26&N1637M7v+L}X60?HpRkO0x`glwf<6LiJGbIH=|KCe)mqM#=gu zVk!FyiWR;45}-{&6+Am^XP<`;OtG^$Z1?OyX3-&zghpFGO;ddxA`0-ecPRq`D>b_0>#f$YB z;4Z*+V&%>h%>F%aAzxlB8H4R)YF^^_SbH`cNnE>C1BGLGY(sFif9o0EunAU1|n%5c0Uv@72;es@RLh zz4HhIhDDy=kIszZ0jFNCyWglB*fLA9^;kdjnUsF#TwO$g->T@xTmUbJ7OXy>`VD`) zv#0#2^&o{Q`FoCBG2HEHDvB^9A@QEUtrtGj+($O-u7cbg@EN}8rAQ~i+sUC=U_1Zp}oU{7Qn5ZW$IuwR#A9)1tTYA{kF~=KU;bs>DRw7iVLaQ{elJwAdc+ zMyQgpzrnYs85pWUR8}91Lc@exQo^eR*r2~j^YSbi%y7nXgzEFo?KV8;N##e9%`A5+ zu2F!kj@BsV>^Qh*j2I0V)}yhmgD=S{7!Dg;;To?Lqvh@LBda9Bf8w6SzJXYar@}n)d9F%Sa;7$n*`@x(2IMB zkbz<2MEJ*eI{AlaI$@~vpknT$L_G3bqKYSs+W&SpN8*7FD8uozzL9=VAibKD zNnXI1f=qMKoLY1}%B`MI?FOC~Wp4G&Qgh}fW5znB6L6&RnyB5Weq8%O*&|+B0lWY6 zcVWIWK-C50tWTHAa7WJg*7=oAe3XCB+^S{-!Ubw`O|TRnUai(t6qv*Olq$Kkr3G;C zxq8RSXci9@s>l0?XJV)7d9hOeW-xF1RmyX`2R3;-M`+SZpgF5r!S*8)A#~sEjSagI zENGFcvRG_{t2{+F+^2GoBT16h>mD_KyKBe2^A;H{`hBu0mm&i5)5@}gI@J4WL_cmW z*N;6O;_^nEtHApt)_T`eDXKiQiI01diT9=Eo{ee}f&F#=M0C|0_*mub6KtDAkDFS4 z_X&M?F@V3iDTbQQOqMkiP;we2>G9`PP8t zo@6@Fm%~^R;%oI_d=6d5?N2(M&cWM$iCgXGt3g@v^ir%pm0vq^=y&z5N^rORd|RQD z1QjWrulMAT!0Jc-KEG!ZaI}v*zy1K#_YS5jJzd`ip4?)c%(boHmq*ApWZwe1=Ld$( z*~Va9Vlp~t=LEiZ^Yt8a-3Densm(gjTZ@tbt{Vf?eTkSxJf4{_4VpUh!n3sVsBV1a z?-RXI@M9DC5HCOhju@}%61zcgdu@0Mf;z!PDB(DHi-0UEcizT7@4^0WqC3v;Qs)lO ziF0MK18|~5K&F1C87rr(I^}LM$eVk;8aXbwjCP{F2kg>oP_5v*lxhMA6JCCMcGIsN z_Vmp;&Wmng^X@|aaq1rJ(hpG%ZR+01g^8;bx*BYJJJJMwRK<)fCL@yy-$ zhNgly@XY{toY(wZsyhn{tjlqCuav{=OSR`QDl~-q9Xxl%Ew*s#`|K&h_rq9mesS#j zk-zwMN+Y)H>>S)_``#n@hpN|tkI;&c8RXB^n5ztQFQIw=`o*`8$XI57 zDtLP>I6Nz=>|$JpZ%!5BgF7~$y!l9>>_8kGacT2%{XPMIHPUVP{Hvhr$`zwW60=xm zc%Xw`=L;;*xt!}dNyKBC#^TX?6LBQXC;jT>31rwG?b_8<1nlCx`%Awz!EcU76&F5^ zz$dScn*8P_IOLlVWFW8#q8p7*LqF2WyOcy6>=^7oqbKHpRm?MRqs!@S$~cXDc;mj% z^TZht5P4v#IX?nHAH<(%^KW3jO3t3bu6~T&nz%T2uM?#tq6VkM7lDPV`jUXrC_021 zzc|{JhCem5Z-1ovmH!=-v;5FV?JMUT*8;m6fX?_y3GL1zkkRgySKUj^bF42{sT~-G zLIshO(tShVdtHQGC#3>fY_CXW)UTpruvOa&BDHUQdgj)w84nBt%U$jtNYD}PyX(^4 zE+o%2dnB6vf$a_Jd=+KM;IA_C7-~k4rl8_-lNALywe=l8+qZ*AI6=6YiwyBmmLiH< zgZP-4(X_U&3RN$e*T#B`!%+^_4*^kAP+>Xke*1R~jGrbxQFiWx_~#4{zEC-~+Osco zD+wb&R#=O6=bOdsJf@nV_)XY3zY(MSu>ri;n26>2qd@svuAXwAggp_=ECE$z7?c`$ z@f~+NxIU~7?d)C0%t7uLUMj!E^v!DMqvR+?MYR`RwV6kDW;u`Zf;n*HereuGR((B`(mIRvvspBL^A zOan)+-M~8AEY$7#uJNor8$|iqMJ7h6I;A!Y5!}I92?Vi4Vi8uJ9Iqn%7rr`YVi#0jQX`nGu;IVSU5uHI! z-JPnBm?{-TZBG&*%9OwC;#Df2OV3&As@sPpskzTxZI?k<^WIQVz$BEI$Mv2t=z^gc zn%3CSb^Nt)<-)OUYVJB4>*I589pb;v$;uu50A7K2KipFYf#-=a{{_1fkyZDIkKf@{ z+}mX@7{5;4kKWy9v-Wuv(iIL_gi`&zO9E^zOxYx4)&G!o!g&A%S#^nj_S4D-c-&U; z)FPvsYzj*;RmVK7OQOH**oAJMRt09+3piqOBTwaQKRyjSI*;i@HX1d17 zhEzUv`~MZ3hd-6?8^;L^Ba$d&hpY<8zNo0Aj3|}9vO-icLaB_BRI-{>MneN-Q(VVP z_THOw%!706{d<1@!8zykocp=2>+^ZPS>*Paf2U*6X}PIbnL=!P!2G0@?iM;j=g9z)oPVWJyy$4hZP!zwcj!gC7@n z6xWYnCWm~HqgOR%GRu;zG*@7ene3^iwn=A^^>g&=MdEv9$yR)MUx{obzv4Q4i2Iqr zvf*ALH%@R1@OKethT82*Y2w7ZLh6*XtlU@)>fU9#1N(Y$-|?zM&D9AUG~(3tp*A4D zu?M@ILL+=)KWqMtWs2Z$3xX_lYVmZ>j}Iln>Dc!%ko+%a1XRi7q(q5SnEAkV>YQ^c zoR&R6$(v1sqr0ws6`cMDpjjxHa)gGlVg8y`#t1=U32H&Da}aaJ-d{iJ zCN&3Td7jL%zq^rxO@{kz#~N-??9&LcZvop9lT5?nSordS)pv8z0;XTN)OOfv0KTcZ zCQB{!g6(3}>C>L`xZAPl;9K=I*cg~~df%2|*s0Os<4^3#cD4R;5>p<9J^?h<)gOUI z+S0wA1mbhY8U~lo4PdLFs*LluQLvZVvMt7-8i$s)O6FAe;n?YTL9yg9Ji5=!K;L~0 z?@#a_lRDW4tNE0Uc%M2HHP_p4yoH67n=*Inwm}+*XwxX&A^~t(v&pQhe`q+Pd*mUU^Pd` zvLyqNr+CbU9;#SGW@>{+gYF;#=YC7x;VJOUo12y-Z_v4ru0I(P6b~(mhgt8v7zTbL z2C|9(!KpveTub}SKyvs?ZQdWd3T8shf$VGpXl!&pIB}f<_DTIk9>4Fry;Y66rFF2J&vDGLt`!6>De)n}=dsx6c{{4MqoGqsVCBF9m`n@oDg1vP zc%Pj;|7{u+PYG0Ni4DS3L#1#r*AnLb7n8y+I0rJK0;STP-C$?hI&JI7s$=u<>h8mz z7<5#omA06l3BvQE>6S&|D`0%TXq)x!F(~pfxj7b91%Kn5KI79R)GJFZt1O*EEB6(j z5XwzR+&$XFk=}z{J7+~18Y)5Z(0|+1XNjETd`;t@q%p7ywjeDYBXa1+Hio)H54BvG zB-0C2q{$3rzSF0pk%B^Bo=Yp#20h7h;2K4x22KNJpN=_>)j6J~V$?~hysN5wdJUp6=~|F;K~x`h(8 z%4ncykkH>h*$Ysxs9t^M6<$=@80IrP0uAN+&o=L$!0NZB?6?<)(JSoEmGUMERR8tQ z{1e7V+UD|~i`V!wkYl&hz|kftd+_i>uF>i4LSCN;69q~Uas-Lwl zOWi;^eQ*3>Svnc=r5+9+aVQ5K*K>*r)IMy8@}4{?H;xBlVtz%G##HKVBHC6oHhsES2;3S%;~uA`u`EQZ;d@;@K9-+Ucv~?@oX<|a zxyQ(0dbhcqy!;U?ehOZ+@EOFxX+AwYpJM!wem-_Ej(8u+pMZLTD`h+NNW_ZB`MfjNFw>F7h>R?My#hgt5YUfBMZhdqj$eom-2 zLCeotN2x3_Tz~V?W2n3i4Ef8A-A>UkvG~}(%0G=@de6$V%8cms=^5<0VbOuD{LHSj zkX00XmS9h((P6GJQ?D#K6I$I4@{}rUB#lSiXzP8>pp&8R|CH@xH;$0AUkU{>=>+>% zcmI)W1#YADbB?rsC?a4ax7qJ6tUXS6b0L+;T{~;Tw%et^n)zWp(hdflC6ONufd%8l zdp)zubXy-p+_p_SLi7wFTVyDZ1_OtpB)~ z@~veEE>{nFDbNA(D=a!$tprD1Yna}3y#VHO4ZbfXQc&Zw*!o1n2GX5|o0nuY8-P;U z1wvfB9WacuJ*lA6EY z1guxmRNnUfh3W~l^w-P8b5YB;LJFV3jTiZx9xQc1bBxh;NkyXPXESnC^LQRa1PPsg zL}`JxeA!ULy>j29-T&Zc(dmA#2do6>zuRovz&%@XslTSw>AihOn!F>j^ zCY9o9_ArnF{jbc{r;WiRzP3!bHjD0i&bvwbj{;-KC5fHRgSgjpJNyDT~YMRF{5;kmP;zbuR_q@7c&@lTU-Xdw3!yeiVb{{7B+1 z&9LA-XCk=RitbyKdXgNx;H0-%RMu4rs$TK7itH@KWf9A_UiX+ux6a?)aEoaax7S)R zXI1rM&xbKizbmO&9w^d2@^c3KSUl*}XDBFOX-`T1FbBpCZFPBJV|Y*}Z)Z2ruWCM_ zO~6ZQID4!z{_oWR(4Nt&4r9rLHPI_cS0DC)bj-O+^i~qAxjncw*A#%eK6J*dE7ilo z*OQ+OR2g*=zH#<2n0KRL#)dmOHyCu-wm;^0PVhT-JPp|$tXe=zSST$uzYmz!Q!Nw1 zwvcw!E5F}K^j3Z1T{NG$k@4sOSsw)ogU(!6&>x?+CTw_@`LsXqJuv{?8(ty5RN3jtI{$N>i;tUO=>V%)$-<}7HXTKHR zrEb!B_WG&JO^GJF72o@r%Ow#^Xd+#4T2mnZu>Jc#$thU+cI!-MeJRZHtJIWgPU6** z4%za2tMI_|>*)7N;=XPmafYs535tUooE$`lkpD8d$nZxJgr9VtTimpYw)`CNcQb~- zORdhB9xw^Ik@7jdv>Z~V3>jsUMQSNyB8Ayn1=lz3!- z3OjCk|7$Jlg&_Xu>^qO^kXyn$(73e^#l>f)EXk#4!57q)!8wVqJk2-kNvwtNgBvFJ zd-`DT^3ihi+-#&N_T<6?*PHkPl>};OlpXf@R;VqpCo{?9ImnA1%52 zxeyFva&~etd(IM`>+tNmlhuw%dn(^+Wllo7;;nO=nTVcG^{*#4xLT2;5bnIC&Vn?f z082A%2$S5cj_XzrfLqUy>?f0DjNdQxGk$aeb6M|(WSCT7#bgLmDAC_aerqfINQsKf z@iO<;UQkg_uTn(qdL4N1oeOnJ?|^|nuM}#+r?AB8{m=i>dhyk@mRoZ+G?3MFJ805J z$6C__or3lz(55Na#Xs-GW6g(E7F`M6*J9nhGNPZ@zl&=|5Pe$fqboiG#5|k0UL1YR zW)uq9Y7&K?jR2ocg6NkIy}&(xK|znV9_t@Q+}EishO2oEq|}eAn81<0v0q{Z+%%GX z%+Czq{sRgH2d+1vm)ZPC+KyRVYkFFHA(#v^45O42#Ql+VEs8u_)5L~k)e_;Pe=oEhaS!i&&auQ@T_cYFb=m*ww1?yKjn~}T1OG`nB z0;@$jTV%TDk)!GD!KL%*Xz5(#lifRoPe)FbBwm?V z%6sgr1Q*aC4g07R@cJ7eC~4UNM%5x|;n&O1;Mw1La>x>1E|>I3ayUmV2fb)vo%h>?8L51NMiFO%{k=urK5#1ya#EL2>(H z$vifmZ?`nN)d~B$C*ykk46y zEl_<9jxrY)SQyf9zumn@U$y$6@+KGWC#!Xg3t0NKysHI%rO0J2@Gn3?TZf6Ycp=Qr zs${&iCga|gsz$GD!iN2Zcl{2E*D>gSkNFPqg}-QF>4K1f}8x_vsROzkI_+6@hu^zqj|{>{1`r+%Duzcj^YO zY~E)H8%P*IbCs+-)`!aBvyTnaI>70rpXud815lAgiRbWV(P?qZ0*c2V@Or47W?C8m zl46eC4HG(eRj}mHnfqXC+8U2beF8R1+SMizzJ@86+T&3K$1U8V*B_=c2Ty)!6K+hrU;Cj-W}Xwo8G~Fs@t6xw7*8fI|{yejg|0F8S&U z$-Q)N-meg1?D-bQ^&NjIJ}SeD45!bf(1{$8`xVbYK1R~Q1MPci7TvHCQK8^&*bZmC zLMmCg=1`Aik6st&I*J@4mI9ppxaz83@!k0^+@UmoI-^k!XZLkzl^$7!9lI8%PTc+r zX5VeNUR%#$chZjFQO;U)GaNms{dWdAp5^#;jhA8?{SzZ4WDeBgqoy?Mx}b_r_1Jf= zAzWiOiu0K12bb(H=T`&6ST48oU9iAL(xa7z59j|(qcX#BC*E^iAX;e3y_7i&0sbr! zzIT?v&q{j7Koj9L>Y$pd^;AD)~OKvTj}d3!aE=o+so zsgx5ilS6`xdB@WmX%twz`^$|#p%)6%5@WrL316DCxlft<2<~K4e&+K3dEfy{eQ0zl z9GDKCww;@T{zS!c-=B?`AgVjM@7oaM_{rNX8-Bz+Ocy!#6Ze-8&5@9RG%C1Obqa76 zP2%;zS*!QUsqk;x@?1|=DGa?^`K48nkB@7#q~g*ip#cg5yG;8*QI(szz}y8d20hW4+XB8!Fu z&nJsPxS;I*NxxwV+*wPTzC0O+UOz@H?N|NKC!{2MoROH{E;E-+UnoE=<^6#hRi+?T zu3tYXXdI;931&n(cfz(Ch3pSV-QdXVu+@O@%Q19Bsq}m$a!!vAwDd1+pzqr{j`Rdtoi z#C{nr^8WHPnufhfx(R_E>u~7VT7`>IGg?@OpS^3zNZNGs)5Pb92~6MogGZ%$5Piu3 zS4C8bIpsb>uek3N+E+c^F|ofK?B7?PbQ2`QrITdGo@_a{pK#g zt+d#Oe{;!Q#<=Ok`^s@IVMVQ%gSBBA!-_&D>dr3VM2R?|g0KrKJ;Ra$B_O!)t^-NjK34G!aipHmjmmAlJ)*RL61x!!oa zlIV#)Y7#15fmEgl8XO@aLs&ntVc7pwJ%jWNN9S|RwS}| z4j%b=#b<3AMJeA)eH+7Q@ciMPl_igP%&+j}4kJ$kP7SZc9v*{+OETOq$BU3BB1OQ? zoC>4sobKtHdNFy8b7RUQDz-oMi2P1?J(gQn#V?xlVPTW^1j9Enq}9ycug#-Dgo|6X zH{lN&k-1%_AwK|jU%13F6qA9!FnRgR*c7BL=s$U8-U$9C2OZ~6endv|NEL;lSq!0h zmF%}{gcd#t8n?$3%tpNK3yq=S=Z8l)ew$RGq+F!1v04X=m3rL%thfM|ICj_Ge@w^c z(ndaWwj=ONHj8ta+>EulUx(~N$@pi(gKpEyR7}v^dQN$15Km-S@3Exy0B6mi>1e4c zjC!0ByqWOcJ-EBGly91bjHRc$D^F3t{SVDauc!kyMo7y{3NK)Cs|`!=t_4`~l!<02 zXo48)q!*6&hT%wNzhdxxIyjQEMyV@A??^5}B}0i$^9?ZBp?Y9` z`fbszW*Jns?B&$PtGE_4Pz5aZfj4p>=<^7xcahuIf&rjDE z;0jBPl)pCNG4i$Tzf?lT;3hWu`WVqmY!mmMd_0Nj>&k3)Tr^nNEF41@ti_2QaShvt z)hO0#q*OgT3cj&+ZWG;0`1-C;Cv`9$w?}FWUn2JAua7y=9go$cLg%iNwJ-W`tJbNK zs%(PiejI%0`}TTpp({8W?#z@St{;DqZ8hTwA7FRP%#v!KhP|0SlY6aP(`^?oCK${#`A zZ8)rw`1H&cH@&9d7unG)TsaAHNE-^X&oJuvEbp`%jV{J975Nm+Lo1Nm$B>>LIgaBG z==!yqME`#?kGuAK4qhGBDW|H_LF|^_^TwcV!XtbB^Y`2Vtk1nz<4Mdjb>UT}@?pfh yu_DLfb#VlZ?+gujs*hoplW5HI$HY8cvfE5Ta0yf7PP*sn(O^AUTonOx{=XO7@>v}z|=i|zLIMu2KG=#x9QwPQV4^X5(75^k*6ZP^2 z{g{=>Sn^jZL3?j6vd`)#bSlr`TM2gI%6w}1Gvya{8N`l5;X9r}hQ4{&`7G0YWUc_e z`XgvD+aNdiI-}N}xs5QLdJux9^c5?w=Y4NZ< zg?Z4W);hj(ya7q7=L0iO5b@1<3UcdtLM9XAE;%h`o#T)k+hzRLbSwHWN=cXwZ4r6Dj~ zWPZkKHVAErrR@iJ3n6;fWXlfsW!T~M=XOZ>FdlXeh~aVV1@CI5muWj@F!V!$yU5X8 z^t37ez;a{-=anqt4ASP|sBUb3Y|9MDeYc={Cf|)hS!q)v1-0;eSe*IpuW0z!{;TbG z@Br2fMtzU)M4&%E=w_(hiSIjSPfi<7!=B4|CPk|icw5wZXB|JSJbG6#H%+GFzYR_i zukKv9en%!fk=P6S>b6Vpmydz_rN{RL7t-PMCFN4ZQ@ucat+jH$X(_PP?hzFk8^KSX zl0dq626+u4yk+ka;jAVrlfTjuXz03|+}pc?NA0AO7s(5lvh7E$2k#;>f2bp^R*yj3 zW5ba|hjA#G!MUSy(|9F46!gs^7z^X4 zkVT5~6iY!neqp$Cw`FV`^CA<-nr`#x!}^G}^i(bC2I%S89VcU0`(|UJaS6KgWNh4@ z+<@5Ec?%b>5MgB6n2n~f4^!1c2P~}yaeYJL<=l@pjAi9tO!B3Yf4|*8pJ+}(DvSQc zy4Bm*nD71RWmy-z|0sVg#+nRHforyos!hlr)T(g)-xO-Rt6&uKsE6T)e9fKP=RxVM zoX7aHIy~R_KF_Ul3`HAwB3?`7g1l(TQ>m3@jJ;rGMY~>!oA0b`PqfVf_t)v-{Ecz2 z2*}(#Xz>>fU9YT!d9T1r5<9u|{RaHfRVh3nKLp1o4^*!|U&KJ>o-6xzZ=k>T;eZms z1zg z$j$(-gL16nBgAQsp5~=LVQ{`no8|D#EcWajw9D923;J<_H=5p2%O~w$esHU)6_nDJ z5B}2{MV%`a4}>%q;8V^u`AWkPT#>8yj`m!Iqo--SWK8Gqvf}rJ9X56F^=N5qiRBW! zcILHT%^k2IPmWc&?`LE?T z7D3V6FtEdM3{x(149G|wVo&*!VIuf;J);YZvmH% z{x@(`p#gO&f4%cMlm}^x0yd%g|1kSv{-9m|A|B=!>Ff3-Lym*oF(U6Y+Qi@RWb7rQ zcEZD&qr@06V%I)$`*b6w-RZTWfdTj?tUOyfF%FeBf^^-i!_X)T$xc!RSZCi_751wS zgim}{t({*&zBGcz3pWzPyf_>1d!!9b+_J*EUT;Esz;8vdO%gEXxLU@^mt!G=Ao=@5 z0QhN{1dWkq;pVWOSk7)5LfEE>%oDdF6ulJAyNuIhEmC^zk~9~usE4F^x?gnk4vyeunkc4o3q4@NQ1#-rGv(CvJssWt{n?M20BqbBjUcavjk z1{p^umNs~oCNX=$je6s3F08r`)ch&W0nIgeM>pnK$O~le)Z0S>u|fJYyF(rDFu{IS zZ*&xF={0_cb0))ZLaQQYaTmNvHW>1%o4t@d)~C^x{luE93YENo z=fRv8Z~K9@z5ZG9WnUD0dHO$wgMH|?^e9BcsSzr#u2q)$wnJWapjrGz6{sgsnLpzp z!E2L%(`tF+C~fn?t%qg?`4&ZfTBOW?Y4omdWkf1@L0^&oUN4M6z*$q1kit1w)%JT` za3~!&Z6vNUY_9``7Kb-)j@E*iephne-9$V&Tu7JbzW{q@I1=63NpPU1F?oki8|HkG z3{%!zL8ouejLei$P%*B=*T!KM{of=BD#-;ydg?U^lp-O$@ga#+o;jGgIP6eC9D=NO z+!w0&hk)_|*G?2A!FIozBqBo_#PyUvGj;8Q=#kIy?sE(9Blkg4#hC&;Klr+d$6x@A z8U>5u+Ga4BBrxrK*&0Wt;BQq>mC*UY_b4O9!QfG zs%e5F#=^zNdV29f?DVxCtUYkL^T>@IizC2t{qhqot3_x<*%qn0jSyr)KUP%QhlfNn zBKGS2MhPA2B!87zG?b8-uvZ!ey4MP4Z#YsB6pum6nlWX+?eXvgjam4)oRTp9gn_Vn zoy6p0HU~RmEWQ*zo`Gxcn)Ob9tVeAvN6jR5GVc7fa&&^Sj^*POaj z=-M84=y9|{ugOey;-zZE28&E znS&=w4iZ^Yg~*o4a)4T{0qP!<6gCsrAkjQyb%!2r zsp<(&waN(_j*U>r`b=qjXd1GnVI^$Ge#pALoHucD6b2q}PjkoTVdItKGtMT{aQrHN zE!|=}$hc!ZV|x{fXuDmeqe+Kj&9o95KC}duy*D1Dag#x9J6G5$#~eCY747HcCE-3b zCu-O2o%q?YiZt#S5m|#w$J=qInDt3d`z8D!{bb1}WCV8kUEB8KbrZ_my!v9v zr4JhAX~cLp$S}#Iy{UYG;$t?^@2)q49GV}mp7_@Q`_J=soFkDyL*ONirwS32&g$Fi zJ{^UU` zNqC`=i;b}lYcnYSU-G#en%G2Oa7$+sl-VXJ;|`={jD~bDyn^36~76) zWA3ku%PnA}U&=EEgJF28U`BP_bR0f8os|6@x(YeXobnf>hk-Ao{%=+H8)zB&WX{gg z0TwPfMh&8Mh$G71ce{~6DZF+lqHGDROZn-PrF$@rzq_*5rxPqhT}Kb(&Vp!K@_EN| z-k5haAZTR!D7xAzclEAUpl)vK^Q%rR$ZFm8xZ_ zlg%pH>y9vHpKbul-I5I!K|^rrXTti!U(EB;(A(66WC_eNP@Y7e zz;_W!>U}tNl+WTsOD9AJnSQ_W-wJx`M~a&z_2RzY0X|ZtgQ%>0u_|UD50T{-aQ4>jTCamQC9 z2?V_VUru0cY9Pk!+GDseG!M-33*iTCe9$3N%k`gZG4#2~7BowhgLPv?>%+z_9J}{C z+@@d-8qOXwR@58CdpgZ50qi|EFBPF%dEFlKTVEdf{$L)jSIGU@2W#L@wa%%^colOG zb~K9JAA=mtTE{=pQ@D!^q8=qg2-~dGN_QTG&zI@{h44;6r)UdbpW_A?OMgq>)zydH zF*5gd50+wPuWj^YmT|1~Q7JKx%Et2rq_roM++8WPJ7I^%05Fe5TD_590lnx`P51Bg zfR#q!BXM&@aOLlTe|98X=+Jc<${q({rJNp)?L~NNXVq_prxT!f=A?nGcn21r%+%zT zBEv_cx@wD@R@ieq#lwRz4YelU3q?~~p>j_QoqpIPPP9B^(!JM$JFc)P2RO}P*PV}< z4pF5b;3uKr3g4=%v5uMKOT`SRgd zb&8Kd>mppe;=2EA=q!q{6`c<{GYPJn2G$?Gw?g&bv^FoLN?Zu5+3)|7gk{uMf7oSK zVehH<7bNj9OGD2` zpj%d`s_$Ap*!38S`0SiP;cm`%w^;hH{oyg5*OIHS-W+F8HO4IOc9!dYt6C%SamG|x z{%*xRSrsL=QY%WfbOjYULp?(ujpKiX^aIn5l=macA1%urpYfDA-y=UxA}D7N zb|qe_0wuGt-TL7~;JeDc+y2`CuCXoud&-}NUnS!*%&)cJgwVA~r;BrV$?)0-pMXu= z;HqJbFdxDL!m%e9+h_^X6=5zHC^_Bba*H&BbsuQk<$dvCD+N{0jiaQo0O)zE=J|MF z7V7eq^5WHparXYn*^l)@5aM5~-_tOPi}GZ(%7Qvj|7soS7*`0MhEao5A1fg2t*coP zw&G7Albywn8{wVV+mc+XHn3hC8XylW;L309bA6P3Js`6Xu9uK8N%Vuc;;sqoNNno=&-gdTliFiQQ{GBR5CjcTW6Ohk0tk$>GXFa>`41 z<^GP}RAqD6S#hiG5a$|_A6gji+g*=cS;GUJm#bludO&JN${hM@a7AT!SK)P$V|!-B zDZl3(n<|O z?Xuo>%mcVu*RCbR-UMSQ^rUXFeaAMvjl^CF5ERXIXH@F>O_Cw+@UH|BtiUpQu$aWSxdZDDoG=!WA? zt5#Nb>#%)~&hI?wK`0bFta)jp4^Aam*L4lgVZ-M!4aVUH_#-Pyv>BU)w@rIm&OV<8 z%V`O+2SXS<{a~7@_ zRLns@wQh@EM?9!HUH{e-u!8)jPFhb6%t2Dx(}U-H7SX8Ik5z%b10N6Ej-tBPgExpT z1e1SwfL#S|l+2kq9R3itAhI!s%=yu-A&h~Tnem+Ug~uq4efBjD@tlYMeo2bgeCl~|+q&TU0~_Z946EqjYhch-H;!o^tHTPv z%wl%S_+P1Qh#!umeB&{gf{gP8Uq4=IMe`ckmV+m2@m<5GGa{PPC=>el+t}?9aAKVd zoX94_ahiB8jj*q1Fi$2FTwVt5sc)zEnfKw-Z@HFCdJA~?xje`HJj%Huhc2AXD;q11 zSCQO8`Z0Sg{%+3Kba2fHe;jpd87~Ntf;pE~(PSa?k6c<7gmdoK46f>cyDk;);?6*No(?9GIW-GH1B%d49{BcnDUe@;^*Jt zz7zlcK~&_>nS~7!>gsO~&rMoFZbRFdOAlP}f?vQjLRArTA1u|q3OU%;d0^!~rcTgj z>&*?kI*KY0Mz1BOGa&y$4rzm}2boQYvdO%)xN5d*{>Sk;cpyA?>hjzSl$ig~TKQEE zf$!Tl`xXb_kuL277wI)5?(X5_I^6?m8?PoFy_*4%j^^v4>>JRqPghj?4jJgQj>oZ| zO2pg7L$*HGXMsqyc7Wj92iwEcf-gQAK+PA~BHlFpa9?e+Ux;lMUz9Rr6tMP#h~zDM zeTFq)Iy`LGb#n}UT>KKPD@=y<-P5z~l5PlSXdJz*Lj+Z?_co;-4e0H? zpLVu)2txh7q;z3!{cAE>*aM*uz7`JRoZC=bO@Etw@WR- ztNH|*DAh9HIHY4g;JkvCej}lo{DbhzjIb0q)en5(+y`HDE7)^5)X;++r)2H0lG*SPKf{UGWXc`A$)r!`p{o{GN`;?kKqYi0KU{l z`qb`K**hA0yLCc-P38>gBB|csN~OX?U43X9&9B%b-m}lM@?vtA{!f z%hP?XYNY}fO^JB%_&SnK-^#VVGzJo%?M9Y0H}K^Ioz*1cd)UVLDLTnz2p_e@B!7O> z4>8Y2;)J893C0V-u7<%YsHAK~jM+PYM`m7vxmFwvApcfof*r?@K3Yv9a#)fT+~~&K;w9(cQm_U+i^^#LTD(4Q;!{ioZ7C z@7ANN+4YOS!F@4NVJHf!8?%(-bY>uaub)I_!z{|^m9g>j*8|TDAKuTeiTJrSOQV1= zjLKpven}0(@IrY-A-pLc!-vI3gpV%ZRCAQ0-QFDFObd{0A0wh{W&O4lNxAVWsW}t&Or0Xou1dhw$9ej=h5bvaWhJ&RU`3H-x z?Ui0eLr$H2-7(APz;3M0t}u=YXS%x$6U*_87XOt}k2&PI>J>@eHH)A8nAQT$v?A}5 zKhh19{;b5aP&3M;3FhOd%&DWsaUlEKOaR8gKd+2Je#Lq8e#&`tIdU9zPLTC0U#(#F z!95attUG|kKRMl5i$jp^IQEYT|X{50kHJ&9hNG>;6BxxG!E<4idD_kks7VpF~rlv;!Ijqy)1E$cvw zf%H?#dLH|BzlltGQi(>p6Sr$f4PZ)W?o2`PFszAuE9NVn#A2eu9FO}7)L52X8p0HO zz8Rw~Uq6oDm7}Eg+I@xl@3ZK>`whdO=3?-HxL#0@OsWoLDuQb~cZ1(^PJm5hhhB=n z7>M3ge;1%K276m=rUGN?usY(X&f7ip*g3noTkc&2zS-qagAqMwdSh>@>yAGVCzV8- z!C;3aQ7oiG&sRW@@YKgzv==3Hc{h3ercvPoS*tT)5ORL+E5E3o2JhbNzw&lr2>0z0 zRO4b8g~7z37OIXqG(7g>js5*4@Zm{P4K$m=TQi1YoPqr~E`5pavwz2{)7gjzD5$nb072=&C!#&^!nkmv}6B<$2(Z`xIY#gR!}KA$-(tNYQE zSkwY0xxYh-@+modd)QuiUM56!(%a4=j^mZ`p^NnM^uETO~us)y_9*xqz0+M{~!3Qr`fo^ zNQR&zYVqrt8~Bl-R``0s940Gyy`_sEMJ2<-n=jN-k!^;Rsg<;W(qg{C%%=z7sAJqz zYIPPaPo}Xfq|D%V2SW?n(lQWxvB;&xw21QLOyRZf-|>{f{)2fnYnZ>huYA*C1%ra~ zzU*b|!oqru*!XcGzE2(Z77uQPy}Mjwi+KF-`JZa3uxcuK6@}&9=1aABZY=NG2@@tl z=rsq^r8?qZgA995*hTOEZnhaQD`<&uCYzfIL+-wZJNsr;3{ zG>21T!r{>=Yba)1NNDsK1eWnQ1?NI)!jF2V+EwLg*s0-QrM;MkhEhs*vN8uC%utOv z#Iga_*r`umZf=HQANI?U_!|soW^NbK)x+q>>QkHGICxdvNXtmSfVq#=Kdzi=Lqgc` zq7;2<`RkRe1`a%pKp>c&aJW1RPt_Vejo3~jxw63V0;SJt=)9|`qd>-@&ul z@UfTKK}C*=!2C~INa;x>&{cfdoKGIXSI?C^2L4uqbV)`r8}}j>@~loWjf_BoUPY*v zLNstk4&8c4IWHV0lXBAbtzzABo)LL*4Mxr#KV-da60hts{y^7Q3kFB@?w&E2Lg|-6 z->;r!m49k3t#)3Z5N!2J{=E4yfRedfN_X!Kz)~W6$V;jbPzVbd=L8oWOMC;L9eyHcJuKD&%-TGv#s?_5KRvgL$aziQxSJ!rT5X&O`y71P7%Nf_RKKAsdbhSQ=B zoO~(s@Wfo;t%xp_yx*bfL)lLT;FH>H_E6;_)VE~N4bF@Mk9hdbn`YH`k~)G@fwG@W z^yD)htyxC)?V1kj&pPod(e{$d^?&%`7HLY{sSqcdhPiTX=Js#6Pm_BW#C6b%9&=ifgj47$PgP58J; z9pyZ-N9-|2YY9HOJ1=uiwiF!ucx>9@$8l=>l=dB$NiZ}i3$Jq`LMKz1r$Tlm?5`>< z`aFPR)oAS?GmLnqHFAjO3bRu*&f4Xs4vkhE?o@c~a6+z%v zb+Z&)gfM0|T?^+1FrcG;xWm35u|mkb+k}WaEv)F9#}sSVI~YCo``4dMH{6;C-?q75jTn zTB*fZM`ia*n5_F;zt=_pM7M z95ZLU@UW)^9VGoeY?~N?p)b+ltaS@0wZmkb`#1@$TU4j?`({BdnC;tOZv}{1`e)q} zn+M~rl<=U-EnsO`dy|iHzaySIPigpH0}}g!x0RKZQSQsBQn__bLEc@C+IL!iahSTM z^Y8R5NS@3-ZOEUCS~na*swwN2ao>~dZCZ24yVk>My;_VVv0XuV*CydN?VpiH%0pO_ zrtMN`F$^Eqoz}|AbD%-*Ix`1lA2Ol4z%Ki)9*^6!{po&K3hM>#0^&?l5bE|V_kMsc zT9OUr!zz1`?f9Cw6X!a*jKRE&W*@M9`LJx9I)ZnDrM_s#_oB7z6Ga(85`OEI>aloQ zjXDqFd5TLZ>+&WyON~b}j1(z;Ee#vQ`kJ|eyH0gsY@?GCBi$mV85nr1?V-%^l0PbU zM$Tc2B-_mXYm}bR*Zt*WU?d9dAEmAl)ixO$*-SQ{Y~gFtw?XaR1W2t z6Sf!sFORDG>9eXgf$<|@!O!OX*r)Zp?`rW7W=6beoYXGG-qns#BVr?r1h*LOc|(~; z-KoA%$$|*4UFEWMv9f;HF zz#BF4G%Xj$faPW0Inj+Yu$+^yq;_7w89wgNmm6)cZ_4+dSJNVtndZFN*t-mCUvAf= zBy?k3i>>ijRU%UL>WzIj*Z}vHzVS2c%g`m&7N|1P1p{ldwL&JV7#aHEzZKH~yr*Yv ze8!c83{Pv-%{SM8E1=J%(|>t&?(^-l_t#sWaOM%>yT;3*KKZ#M|xKvdSkUkv1u5C zMqH=r$jd0)Z}RJvNF2!QKiF4cxQJ}QHH_g)2qyxD-IsUTf+XeX7El?%Qvss4IoUQ~ zl-gOaQrQZVpzc*){5YhX^S5Qqo`lunC_bL!YY=eY_DBwq$SK?0qxn8%aopPAMy6BIL})6G({wzrhR~6N?dWLJi9?Cl}qoYAQ6?%(OtZK zasZ`rmj7#|%-JmW3v$FwOrcn8RB&ALGTc3IpU(9MWxi-qQ*u>p7C&no^cxFXgqN~U z=Q*EKdLSaz?4CmlK(+PR64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h z64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h64(;h N64(;h68L`#{14*XEKUFb diff --git a/tests/v2/fixture/test_format_compatibility/array_13/compressor_1/.zarray b/tests/v2/fixture/test_format_compatibility/array_13/compressor_1/.zarray deleted file mode 100644 index 02cd2d7491..0000000000 --- a/tests/v2/fixture/test_format_compatibility/array_13/compressor_1/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 3000 - ], - "compressor": { - "id": "zlib", - "level": 1 - }, - "dtype": "Lh!kc^ZOszZ`dA|pj9kx0KvsEiaMrBrs5tn9M) zIM?2L?`vQCy7vCv|G$5I-}gP|JkRqv;zANEJA&rWw&#RRyKXyd>}Ke}iw?Q(9A<~$VwZ;2Q|kz9o8FZ}CNL0IuBv#u`_jmN zjQ@U)JF5%Q4_%WK42{LVFqEUUF@lOsdH!8u<>)H!bIt$yJoqqjfz5kt(+6eSCNBHOb?8Ku=H(L)SO~IX*>l#WL z6qFeKG{#et1g%xCNva!}n6sJ}9{|hfUw`|M#DCHFHCF9>a>M{09*%;>;6-rax#1qB z(1{eGnuHkZR=9YKe*f*{IW$|AO&?UQg;W1onH>ApiO%+8WeRO1G_fmAtczomza!8X zPYP~DS>YLbKAtK(;T>>~r)Lrjhc&;<`H}GHg$y^Tx(>LMeZkg4hX~isILzy5)d9Cd z%85}!I{6viqc6O2$Iyf;l$}0s88)8{-2W`y3)}SM*`zq9fF>y>vAnefvdCs%p6uQP zV@AQFf)362#<`%yy0H&L7!L(V56*-4?DPH11N~U46E|2vONN^`B}h8pjenGW>K-VZ zLq;RZ%`}-MR2&yxT!IW7yE2)slQsiBH~g!zLum;6PAb*himrhFCgmOVJtrZEQ_9Ew z#XKq%+1loO*4Ygc&F%L;( zX^xC4A~b%Nwdj)`gMIZv8|#xT*pQc9Bk`32+pC`h*uU(Ad|}?r^h2kYRI zBfs@?16l&*;v&uc_7Nb3R^4J4uEA(p-YUK~n_zbD{Jmp0`=IY}h|;jv6fD_#-3YEA z;}MNT`5mNTSX}IrS9&*uOYv7@3SZ0uTrE&98Jfklw?=<;kK}^w(V*-EHwyN>?=kws zNr1wEr@wZK(#VTT@2n2lKMHPd&v*Pw4Z)Qm7yEa~^n}|7Vl{5_^`q#qy9rtMd*R6R z*)x<|M7*5h|2C*)5z>VCB(yb?!BeVbZcKub+ezLVV6`--hJv@_Zt#GZXw>aLQ}I+p~BLZA_2y=0FwdPxYT05hP$zy3cA) z*dQ`pFpB)`mkj1*U!SG<4dE50HS>vUEg-V<6ZiVZO=Jx`a@6)Z1$12-&Hl=-U|{Kq z!v}loQTrB$v+j#gJQIJQOx&jl(wHgjmeytX=s=mt&cpRE!XKYxSvCu;hFhj@14!_8 zu~?Sq7Y1o392LFIdf-qPTl?udTQH}VL%ESV2K*mabhIlP@zc(uL>}sW+WsuWQg&wt zT)l23vAb>>U;1snAn}x8hf+rFUC$+~%_={1=!1(Qq0qjXCz(8X<^rggkeI1`k>_D1?9@GT@yyUNaF;!cmR((fo0}>f zdCsFCJak!YugX724E}6-#Hs^zIIEB6Z;n986%KS-ls2_^Q4|)jLQSf21^O2Jw!`tkNNS*M~XS7`?Z&t{eyl zE7k&kA1;7|^Yt0EDij#k9u;<)YQYNCrT1+&rl7pct}<%J1Q5c>+0vnfsIy{SQ+sm} zg|F-7^KO2GAMPDCr_WGer!6~tk{bsGu3s){ zL`HIuaMQb*3ef9fnJ7+}g=>k!$Dh(p!H%+%iJK3yfi-4cp;({+1+M?DiE5cam+hMN zyd%{(xFcFUU~wKPUky(UEw*5`*9%Rxk#&5>99Mhk&;l|k`Tr18n}GKQdpO@^jRME4 z^dEbNM!Z}ky^`RQ2$Eb(&E_^0P%F$Mx$R{SFeWAa&HLF31S4zCXT}RCN8It>GY@*g zkF^I=!b)TmH1TIVR8WMx8H$`uS<_%4@9+I~s|IeRL=b;c|No@q9(IM4N!-h?&8T)b z8RhH=p}&d}p=-)l+)0##<4Ly@}FwDvoML9&y{6Z zA}g^t+VJO&e+%$ZSt`yln26hUiOv-@zk!LU^y>*MIT)sstnay_9KCIO1=*8Vfyp6P z`{nm0&=;d9zT#RzFAG~kGwWqAr(r+q;nx6ndqQ4BklOHY!!5=e{tLj!5+}tpLBfYI zm1HNYH7LDgz^Q&lf*!msfh4r*&a>8N99UPg$wR!~{3nurK>& zv6VhCAhjnS)7I8I1%1dE;=I!BGhK#RtV#l!f=u#nRK8~{|H_BlW8wX^8U1*yC^nb5 zy%J+~*>xSd(}|h7gUkx0WN;3@@_pf82FCNyS<>dOfUs5Ln`PN%ycb@z)OV;JcV#l2 zvPqsq^NMYt^RXT0WlN8z=B}aU6lJ1g2Q5KOmUBgGV-U(G%muWk)-Y()E92%`2b@ch zUEa$;#DJpXtiM^O;p~!!3#<1UCh$@Qd-(=IcRrtf=3EQxOInnFb+!#k=9}-oS(^o& zQoHNAn@Mo_*YhvJzVw7ezt`Pj`Q7kO=E$-4HqG$6G;^v?qYi((ipZ^*8OKA+r|+mA zYQwnW+ulg&EWy`_*WLrqrqJMJY)V%98op#4rKfdVLUszh_4R#W*!iris3vC>Mh?9# zbq}YNPf>iMkz6|jgPK~ScTHz-Dx>lVfo2jVlx%3b87JYGbMM^!>kF{ZGNydNV+haO zG`}iR$?RH$ z{n@p}OXPlRYqgmv7$sxnSi^^Jv3)?B_Rk=0e*tW4_Xzq-X~x71iC@Ot53sC(UD7RU z6^o)KUQP~lz>^-=L1p_Dh)&*F^t*QiOTCxQnK+N*@rXA~@7PP>jF!hkgJMScDWwAv zTt8;PM#wd_^>8MVwr#Xuy%GV5%h@b;qI1xd`{r2Sp(%WLnqTgr_aGdR>w9;we-