diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000..ed9def7a62 Binary files /dev/null and b/.DS_Store differ diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..034e848032 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,21 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 5.1.x | :white_check_mark: | +| 5.0.x | :x: | +| 4.0.x | :white_check_mark: | +| < 4.0 | :x: | + +## Reporting a Vulnerability + +Use this section to tell people how to report a vulnerability. + +Tell them where to go, how often they can expect to get an update on a +reported vulnerability, what to expect if the vulnerability is accepted or +declined, etc. diff --git a/zarr/core.py b/zarr/core.py index e5b2045160..b978fbb1c6 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -1,15 +1,19 @@ import binascii import hashlib +import h5py import itertools +import os import math import operator import re +import zarr +from io import BytesIO +from kerchunk.hdf import SingleHdf5ToZarr +from zarr.indexing import PartialChunkIterator from functools import reduce from typing import Any - import numpy as np from numcodecs.compat import ensure_bytes - from zarr._storage.store import _prefix_to_attrs_key, assert_zarr_v3_api_available from zarr.attrs import Attributes from zarr.codecs import AsType, get_codec @@ -235,7 +239,7 @@ def _load_metadata(self): with self._synchronizer[mkey]: self._load_metadata_nosync() - def _load_metadata_nosync(self): + def _load_metadafta_nosync(self): try: mkey = _prefix_to_array_key(self._store, self._key_prefix) meta_bytes = self._store[mkey] @@ -966,7 +970,158 @@ def _get_basic_selection_zd(self, selection, out=None, fields=None): out[selection] = chunk[selection] return out - +dsize = (60, 404, 802) +dchunks = (12, 80, 160) +dvalue = 42. +docstr = "r" +print(docstr) +def build_zarr_dataset(): + store = zarr.DirectoryStore('data/array.zarr') + z = zarr.zeros(dsize, chunks=dchunks, store=store, overwrite=True) + z[...] = dvalue + zarr.save_array("example.zarr", z, compressor=None) +def h5py_chunk_slice_info(): + buf = BytesIO() + with h5py.File(buf, 'w') as fout: + fout.create_dataset('test', shape=dsize, + chunks=dchunks, dtype='f8') + fout['test'][:] = dvalue + buf.seek(0) + + + with h5py.File(buf, 'r') as fin: + ds = fin['test'] + ds_id = fin['test'].id + num_chunks = ds_id.get_num_chunks() + print("\n") + print("H5Py stuffs") + print("==================") + print(f"Dataset number of chunks is {num_chunks}") + print("\nAnalyzing 0th and 5th chunks") + for j in [0, 5]: + print(f"Chunk index {j}") + si = ds_id.get_chunk_info(j) + print("Chunk index offset", si.chunk_offset) + print("Chunk byte offset", si.byte_offset) + print("Chunk size", si.size) + + tot_size = ds.size * ds.dtype.itemsize + print(f"\nTotal chunks size {tot_size}") + print("\nNow looking at some slices:") + data_slice = ds[0:2] # get converted to an ndarray + print(f"Slice Dataset[0:2] shape {data_slice.shape}") + # use zarr.indexing.PartialChunkIterator + PCI = PartialChunkIterator((slice(0, 2, 2), ), ds.shape) + print("Slice offset and size:", list(PCI)[0][0], "and", + list(PCI)[0][1] * ds.dtype.itemsize) + print("\n") + data_slice = ds[4:7] + print(f"Slice Dataset[4:7] shape {data_slice.shape}") + PCI = PartialChunkIterator((slice(4, 7, 1), ), ds.shape) + print("Slice offset and size", list(PCI)[0][0], "and", + list(PCI)[0][1] * ds.dtype.itemsize) + print("\n") + data_slice = ds[0:60] # the whole cake + print(f"Slice Dataset[0:60] shape {data_slice.shape}") + PCI = PartialChunkIterator((slice(0, 60, 1), ), ds.shape) + print("Slice offset and size", list(PCI)[0][0], "and", + list(PCI)[0][1] * ds.dtype.itemsize) + print("\n") + + # kerchunk it! + ds = SingleHdf5ToZarr(buf).translate() + print("\nKerchunk-IT stuffs") + print("======================") + no_chunks = len(ds["refs"].keys()) - 3 + print(f"Dataset number of chunks is {no_chunks}") + print(f"(0, 0, 0) Chunk: offset and size:", ds["refs"]["test/0.0.0"][1], ds["refs"]["test/0.0.0"][2]) + print(f"(0, 0, 5) Chunk: offset and size:", ds["refs"]["test/0.0.5"][1], ds["refs"]["test/0.0.5"][2]) + # print(f"(0, 5, 0) Chunk: offset and size:", ds["refs"]["test/0.5.0"][1], ds["refs"]["test/0.5.0"][2]) + chunk_sizes = [] + for val in ds["refs"].values(): + if isinstance(val[2], int) or isinstance(val[2], float): + chunk_sizes.append(val[2]) + print("Min chunk size", np.min(chunk_sizes)) + print("Max chunk size", np.max(chunk_sizes)) + print("Total size (sum of chunks), UNCOMPRESSED:", np.sum(chunk_sizes)) + print("\n") + + +def zarr_chunk_slice_info(): + """Use pure zarr insides to get chunk/slice info.""" + zarr_dir = "./example.zarr" + if not os.path.isdir(zarr_dir): + build_zarr_dataset() + ds = zarr.open("./example.zarr") + print("Zarr stuffs") + print("==================") + print(f"Data file loaded by Zarr\n: {ds}") + print(f"Info of Data file loaded by Zarr\n: {ds.info}") + # print(f"Data array loaded by Zarr\n: {ds[:]}") + print(f"Data chunks: {ds.chunks}") + + # Zarr chunking information + # from zarr.convenience._copy(); convenience module l.897 + # https://zarr.readthedocs.io/en/stable/api/convenience.html#zarr.convenience.copy + shape = ds.shape + chunks = ds.chunks + chunk_offsets = [range(0, s, c) for s, c in zip(shape, chunks)] + print("Chunk offsets", [tuple(k) for k in chunk_offsets]) + print("Zarr number of chunks", len(list(itertools.product(*chunk_offsets)))) + offsets = [] # index offsets + sels = [] # indices + ch_sizes = [] # chunk sizes + for offset in itertools.product(*chunk_offsets): + offsets.append(offset) + sel = tuple(slice(o, min(s, o + c)) + for o, s, c in zip(offset, shape, chunks)) + sels.append(sel) + islice = ds[sel] + slice_size = islice.size * islice.dtype.itemsize + ch_sizes.append(slice_size) + + print("\nAnalyzing 0th and 5th chunks") + for j in [0, 5]: + print(f"Chunk index {j}") + print("Chunk index offset:", offsets[j]) + print("Chunk position:", sels[j]) + print("Chunk size:", ch_sizes[j]) + + print("\nChunks information") + print("Min chunk size:", np.min(ch_sizes)) + print("Max chunk size:", np.max(ch_sizes)) + print("Total chunks size COMPRESSED:", np.sum(ch_sizes)) + + tot_size = ds.size * ds.dtype.itemsize + print(f"\nTotal size (sum of chunks), COMPRESSED: {tot_size}") + + # slice this cake + print("\nNow looking at some slices:") + data_slice = ds[0:2] # zarr data slice + print(f"Slice Dataset[0:2] shape {data_slice.shape}") + PCI = PartialChunkIterator((slice(0, 2, 1), ), ds.shape) + print("Slice offset and size", list(PCI)[0][0], "and", + list(PCI)[0][1] * ds.dtype.itemsize) + print("\n") + data_slice = ds[4:7] # zarr data slice + print(f"Slice Dataset[4:7] shape {data_slice.shape}") + PCI = PartialChunkIterator((slice(4, 7, 1), ), ds.shape) + print("Slice offset and size", list(PCI)[0][0], "and", + list(PCI)[0][1] * ds.dtype.itemsize) + print("\n") + data_slice = ds[0:60] # the whole cake + print(f"Slice Dataset[0:60] shape {data_slice.shape}") + PCI = PartialChunkIterator((slice(0, 60, 1), ), ds.shape) + print("Slice offset and size", list(PCI)[0][0], "and", + list(PCI)[0][1] * ds.dtype.itemsize) + print("\n") + + +def main(): + h5py_chunk_slice_info() + zarr_chunk_slice_info() +main() +if __name__ == '__main__': def _get_basic_selection_nd(self, selection, out=None, fields=None): # implementation of basic selection for array with at least one dimension @@ -2541,6 +2696,15 @@ def append(self, data, axis=0): """ return self._write_op(self._append_nosync, data, axis=axis) + + + + + + + + + def _append_nosync(self, data, axis=0): # ensure data is array-like