diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 86f47e7..0000000 --- a/.coveragerc +++ /dev/null @@ -1,6 +0,0 @@ -[report] -omit = - */_generated/*.py -exclude_lines = - # Re-enable the standard pragma - pragma: NO COVER diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 925072d..0000000 --- a/.gitignore +++ /dev/null @@ -1,46 +0,0 @@ -*.py[cod] -*.sw[op] - -# C extensions -*.so - -# Packages -*.egg -*.egg-info -dist -build -eggs -parts -bin -var -sdist -develop-eggs -.installed.cfg -lib -lib64 -__pycache__ - -# Installer logs -pip-log.txt - -# Unit test / coverage reports -.coverage -.tox -nosetests.xml - -# Virtual environment -env/ -coverage.xml - -# Directories used for creating generated PB2 files -generated_python/ -cloud-bigtable-client/ - -# Make sure a generated file isn't accidentally committed. -pylintrc_reduced - -# Local config not intended to be uploaded -system_tests/local_setup - -# Sphinx built docs -docs/_build diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 28876e3..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,83 +0,0 @@ -## Building `_pb2.py` files - -The `gcloud_bigtable/_generated` directory depends on files -generated from [`bigtable-protos`][1]. These generated -files must be re-generated from time to time. - -To generate them, execute - -```bash -make generate -``` - -Compiling them to Python files requires -- the latest version of the `protoc` compiler to support the `proto3` syntax -- the `grpc_python_plugin` (which requires installing `grpc`) -- the latest version of the Python [`protobuf`][3] library: - - ```bash - $ [sudo] pip install "protobuf>=3.0.0a3" - ``` - -## Installing `protoc>=3.0.0` - -In order to install `protoc` version `>= 3.0.0`, follow the -[instructions][2] on the project. For Debian-based Linux, they are roughly - -```bash -$ sudo apt-get install autoconf libtool -$ git clone https://github.com/google/protobuf -$ cd protobuf -$ # Optionally checkout a tagged commit: git checkout v3.0.0-alpha-3.1 -$ ./autogen.sh # Generate the configure script -$ # Build and install C++ Protocol Buffer runtime and -$ # the Protocol Buffer compiler (protoc) -$ ./configure [--prefix=/usr/local] # or [--prefix=/usr] -$ make -$ make check -$ # Make sure that `make check` passes -$ [sudo] make install -$ # May need to update dynamic linker config via: -$ # [sudo] ldconfig -``` - -## Testing - -Unfortunately, `tox` will fail by default due to the absence of -`grpcio` from `setup.py`. In order to successfully set up a `tox` -test environment (e.g `ENV=py27`), run - -```bash -tox -e ${ENV} -``` - -and watch the install fail (missing dependencies), then with the partially -set up environment, execute: - -```bash -BREW_PREFIX=$(brew --prefix) -VERSION=0.10.0a0 -CFLAGS=-I${BREW_PREFIX}/include LDFLAGS=-L${BREW_PREFIX}/lib \ - .tox/${ENV}/bin/pip install grpcio==${VERSION} -unset BREW_PREFIX -``` - -## False Starts - -This library originally attempted to support HTTP-RPC requests -over HTTP/1.1. This is not allowed by the backend, and results in -a 400 Bad Request: - -```json -{ - "error": { - "code": 400, - "message": "Proto over HTTP is not allowed for service 'bigtableclusteradmin.googleapis.com'.", - "status": "FAILED_PRECONDITION" - } -} -``` - -[1]: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/master/bigtable-protos/src/main/proto/google/ -[2]: https://github.com/google/protobuf -[3]: https://pypi.python.org/pypi/protobuf diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 36ebf02..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include README.md -graft gcloud_bigtable -global-exclude *.pyc diff --git a/Makefile b/Makefile deleted file mode 100644 index 7a8b6d8..0000000 --- a/Makefile +++ /dev/null @@ -1,79 +0,0 @@ -GENERATED_DIR=$(shell pwd)/generated_python -FINAL_DIR=gcloud_bigtable/_generated -BREW_PREFIX=$(shell brew --prefix) -LD_LIBRARY_PATH=$(BREW_PREFIX)/lib -GRPC_PLUGIN=$(BREW_PREFIX)/bin/grpc_python_plugin - -help: - @echo 'Makefile for a gcloud-python-bigtable ' - @echo ' ' - @echo ' make generate Generates the protobuf modules ' - @echo ' make check_generate Checks that generate succeeded ' - @echo ' make clean Clean generated files ' - -generate: - [ -d cloud-bigtable-client ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client - cd cloud-bigtable-client && git pull origin master - mkdir -p $(GENERATED_DIR) - # Data API - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) google/bigtable/v1/*.proto - mv $(GENERATED_DIR)/google/bigtable/v1/* $(FINAL_DIR) - # Cluster API - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/bigtable/admin/cluster/v1/*.proto - mv $(GENERATED_DIR)/google/bigtable/admin/cluster/v1/* $(FINAL_DIR) - # Table API - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/bigtable/admin/table/v1/*.proto - mv $(GENERATED_DIR)/google/bigtable/admin/table/v1/* $(FINAL_DIR) - # Auxiliary protos - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/api/*.proto - mv $(GENERATED_DIR)/google/api/* $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/any.proto - mv $(GENERATED_DIR)/google/protobuf/any_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/duration.proto - mv $(GENERATED_DIR)/google/protobuf/duration_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/empty.proto - mv $(GENERATED_DIR)/google/protobuf/empty_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/protobuf/timestamp.proto - mv $(GENERATED_DIR)/google/protobuf/timestamp_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/longrunning/operations.proto - mv $(GENERATED_DIR)/google/longrunning/operations_pb2.py $(FINAL_DIR) - cd cloud-bigtable-client/bigtable-protos/src/main/proto && \ - protoc --python_out=$(GENERATED_DIR) --grpc_out=$(GENERATED_DIR) \ - --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \ - google/rpc/status.proto - mv $(GENERATED_DIR)/google/rpc/status_pb2.py $(FINAL_DIR) - python scripts/rewrite_imports.py - -check_generate: - python scripts/check_generate.py - -clean: - rm -fr cloud-bigtable-client $(GENERATED_DIR) - -.PHONY: generate check_generate clean diff --git a/README.md b/README.md index a629913..4190617 100644 --- a/README.md +++ b/README.md @@ -1,171 +1,7 @@ # Google Cloud Bigtable Python Library (Alpha) -[![Documentation Status](https://readthedocs.org/projects/gcloud-python-bigtable/badge/?version=latest)](https://readthedocs.org/projects/gcloud-python-bigtable/?badge=latest) +This library was an alpha implementation of [Google Cloud Bigtable][1]. +The project has been moved into [`gcloud-python`][2]. -### An extension to [`gcloud-python`][1] - -This library supports RPC requests to the Google Cloud Bigtable API over -HTTP/2. In order to support this, we'll rely on [`grpc`][2]. Unfortunately, -the install story of gRPC is still developing. We are working with the -gRPC team to rapidly make the install story more user-friendly. - -The Cloud Bigtable API is supported via JSON over HTTP/1.1 (see the -[`gcloud` CLI][4] use of the API). However, features of HTTP/2 such -as streaming are [used][5] by the Bigtable API and are not possible -to support via HTTP/1.1. - -## Installing gRPC - -**NOTE**: These are out-of-date (as of November 18, 2015). - -Make sure you have downloaded [`homebrew`][6] on OS X or -[`linuxbrew`][7] on Linux. (On Linux, also be sure to -add `brew` to `${PATH}` as instructed.) - -First, install the gRPC core (C/C++) library - -```bash -curl -fsSL https://goo.gl/getgrpc | bash -``` - -Since this uses `brew` to install, this cannot be run as -root (via `sudo`). - -Next, install the [Python][11] `grpcio` [library][12] via: - -```bash -BREW_PREFIX=$(brew --prefix) -[sudo] CFLAGS=-I${BREW_PREFIX}/include LDFLAGS=-L${BREW_PREFIX}/lib \ -pip install --upgrade grpcio -``` - -You may wish to run this as root (via `sudo`) so it can be included with -your machine's Python libraries. If not, you'll need to use a Python -[virtual environment][13] so that non-privileged (i.e. non-`sudo`) installs -are allowed. - -Finally, you can install this library via - -```bash -[sudo] pip install -e git+https://github.com/dhermes/gcloud-python-bigtable#egg=gcloud-bigtable -``` - -Again, you may wish to install as root or in a virtual environment. - -## Running `gcloud_bigtable` code - -Since the gRPC core is installed via `brew`, the system libraries -are not in a place that Python can readily find them. - -In order to run code that uses `gcloud_bigtable` with these -libraries, you'll need to set the `LD_LIBRARY_PATH` environment -variable: - -```bash -BREW_PREFIX=$(brew --prefix) -export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${BREW_PREFIX}/lib -``` - -## Authorization - -You can make requests with your own Google account by -using the [`gcloud` CLI tool][8]. You can create an access token via - -```bash -gcloud login -``` - -and then from there, the token created will be picked up automatically -when you create an object which requires authentication: - -```python -from gcloud_bigtable.client import Client -client = Client() -``` - -If instead you'd like to use a service account, you can set an -environment variable to the path containing the service account -credentials JSON file: - -```bash -export GOOGLE_APPLICATION_CREDENTIALS="/path/to/keyfile.json" -``` - -If you are **familiar** with the [`oauth2client`][9] library, -you can create a `credentials` object and pass it directly: - -```python -from gcloud_bigtable.client import Client -client = Client(credentials=credentials) -``` - -## Enabling the Bigtable API - -1. Visit [Google Cloud Console][14] -1. Either create a new project or visit an existing one -1. In the project, click **"APIs & auth > APIs"**. The URI - should be of the form - - ``` - https://console.developers.google.com/project/{project-id}/apiui/apis/library - ``` - -1. On this page, search for **bigtable**, and click both `Cloud Bigtable API` - and `Cloud Bigtable Table Admin API`. -1. For each API, click "Enable API" (if not already enabled) - -## Getting a Service Account Keyfile - -1. Visit [Google Cloud Console][14] -1. Either create a new project or visit an existing one -1. In the project, click **"APIs & auth > Credentials"**. The URI - should be of the form - - ``` - https://console.developers.google.com/project/{project-id}/apiui/credential - ``` - -1. On this page, click "Create new Client ID", select "Service account" as - your "Application type" and then download the JSON key provided. - -After downloading, you can use this file as your -`GOOGLE_APPLICATION_CREDENTIALS`. - -## Creating a Cluster in the UI - -1. Visit [Google Cloud Console][14] -1. Either create a new project or visit an existing one -1. In the project, click **"Storage > Cloud Bigtable"**. The URI - should be of the form - - ``` - https://console.developers.google.com/project/{project-id}/bigtable/clusters - ``` - -1. On this page, click **Create a cluster** and take note of the "Cluster ID" - and "Zone" you use when creating it. - -## Error Messages - -Unfortunately, the gRPC Python library does not surface -exceptions returned from the API. An [issue][10] has been -filed with the gRPC team about this problem. - -## Development - -See [`CONTRIBUTING.md`][3] for instructions on development. - -[1]: https://github.com/GoogleCloudPlatform/gcloud-python -[2]: https://www.grpc.io/ -[3]: https://github.com/dhermes/gcloud-python-bigtable/blob/master/CONTRIBUTING.md -[4]: https://cloud.google.com/sdk/gcloud/reference/alpha/bigtable/clusters/list -[5]: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L36-L46 -[6]: http://brew.sh/ -[7]: https://github.com/Homebrew/linuxbrew#install-linuxbrew-tldr -[8]: https://cloud.google.com/sdk/gcloud/ -[9]: https://pypi.python.org/pypi/oauth2client -[10]: https://github.com/grpc/grpc/issues/2611 -[11]: https://github.com/grpc/grpc/tree/master/src/python -[12]: https://pypi.python.org/pypi/grpcio -[13]: http://docs.python-guide.org/en/latest/dev/virtualenvs/ -[14]: https://console.developers.google.com/ +[1]: https://cloud.google.com/bigtable/docs/ +[2]: http://gcloud-python.readthedocs.io/en/stable/bigtable-usage.html diff --git a/_fake_grpc/grpc/__init__.py b/_fake_grpc/grpc/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/_adapter/__init__.py b/_fake_grpc/grpc/_adapter/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/_adapter/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/_adapter/_c.py b/_fake_grpc/grpc/_adapter/_c.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/_adapter/_c.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/early_adopter/__init__.py b/_fake_grpc/grpc/early_adopter/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/early_adopter/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/early_adopter/implementations.py b/_fake_grpc/grpc/early_adopter/implementations.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/early_adopter/implementations.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/__init__.py b/_fake_grpc/grpc/framework/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/alpha/__init__.py b/_fake_grpc/grpc/framework/alpha/__init__.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/alpha/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/_fake_grpc/grpc/framework/alpha/utilities.py b/_fake_grpc/grpc/framework/alpha/utilities.py deleted file mode 100644 index 61e4b79..0000000 --- a/_fake_grpc/grpc/framework/alpha/utilities.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/docs/client-intro.rst b/docs/client-intro.rst deleted file mode 100644 index 495bd01..0000000 --- a/docs/client-intro.rst +++ /dev/null @@ -1,102 +0,0 @@ -Base for Everything -=================== - -To use the API, the :class:`Client ` -class defines a high-level interface which handles authorization -and creating other objects: - -.. code:: python - - from gcloud_bigtable.client import Client - client = Client() - -Long-lived Defaults -------------------- - -When creating a :class:`Client `, the -``user_agent`` and ``timeout_seconds`` arguments have sensible -defaults -(:data:`DEFAULT_USER_AGENT ` and -:data:`DEFAULT_TIMEOUT_SECONDS `). -However, you may over-ride them and these will be used throughout all API -requests made with the ``client`` you create. - -Authorization -------------- - -This will use the Google `Application Default Credentials`_ if -you don't pass any credentials of your own. If you are **familiar** with the -`oauth2client`_ library, you can create a ``credentials`` object and -pass it directly: - -.. code:: python - - client = Client(credentials=credentials) - -In addition, the -:meth:`from_service_account_json() ` -and -:meth:`from_service_account_p12() ` -factories can be used if you know the specific type of credentials you'd -like to use. - -Project ID ----------- - -.. tip:: - - Be sure to use the **Project ID**, not the **Project Number**. - -You can also explicitly provide the ``project`` rather than relying -on the inferred value: - -.. code:: python - - client = Client(project='my-cloud-console-project') - -When implicit, the value is inferred from the environment in the following -order: - -* The ``GCLOUD_PROJECT`` environment variable -* The Google App Engine application ID -* The Google Compute Engine project ID (from the metadata server) - -Admin API Access ----------------- - -If you'll be using your client to make `Cluster Admin`_ and `Table Admin`_ -API requests, you'll need to pass the ``admin`` argument: - -.. code:: python - - client = Client(admin=True) - -Read-Only Mode --------------- - -If on the other hand, you only have (or want) read access to the data, -you can pass the ``read_only`` argument: - -.. code:: python - - client = Client(read_only=True) - -This will ensure that the -:data:`READ_ONLY_SCOPE ` is used -for API requests (so any accidental requests that would modify data will -fail). - -Next Step ---------- - -After a :class:`Client `, the next highest-level -object is a :class:`Cluster `. You'll need -one before you can interact with tables or data. - -Head next to learn about the `Cluster Admin API`_. - -.. _Application Default Credentials: https://developers.google.com/identity/protocols/application-default-credentials -.. _oauth2client: http://oauth2client.readthedocs.org/en/latest/ -.. _Cluster Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1 -.. _Table Admin: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/tree/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1 -.. _Cluster Admin API: cluster-api.html diff --git a/docs/client.rst b/docs/client.rst deleted file mode 100644 index feb8aed..0000000 --- a/docs/client.rst +++ /dev/null @@ -1,7 +0,0 @@ -Client -~~~~~~ - -.. automodule:: gcloud_bigtable.client - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/cluster-api.rst b/docs/cluster-api.rst deleted file mode 100644 index b06aca1..0000000 --- a/docs/cluster-api.rst +++ /dev/null @@ -1,179 +0,0 @@ -Cluster Admin API -================= - -After creating a :class:`Client `, you can -interact with individual clusters, groups of clusters or available -zones for a project. - -List Clusters -------------- - -If you want a comprehensive list of all existing clusters, make a -`ListClusters`_ API request with -:meth:`Client.list_clusters() `: - -.. code:: python - - clusters = client.list_clusters() - -List Zones ----------- - -If you aren't sure which ``zone`` to create a cluster in, find out -which zones your project has access to with a `ListZones`_ API request -with :meth:`Client.list_zones() `: - -.. code:: python - - zones = client.list_zones() - -You can choose a :class:`string ` from among the result to pass to -the :class:`Cluster ` constructor. - -As of right now, the available zones are - -.. code:: python - - >>> zones - [u'asia-east1-b', u'europe-west1-c', u'us-central1-c', u'us-central1-b'] - -Cluster Factory ---------------- - -To create a :class:`Cluster ` object: - -.. code:: python - - cluster = client.cluster(zone, cluster_id, - display_name=display_name, - serve_nodes=3) - -Both ``display_name`` and ``serve_nodes`` are optional. When not provided, -``display_name`` defaults to the ``cluster_id`` value and ``serve_nodes`` -defaults to the minimum allowed: ``3``. - -Even if this :class:`Cluster ` already -has been created with the API, you'll want this object to use as a -parent of a :class:`Table ` just as the -:class:`Client ` is used as the parent of -a :class:`Cluster `. - -Create a new Cluster --------------------- - -After creating the cluster object, make a `CreateCluster`_ API request -with :meth:`create() `: - -.. code:: python - - cluster.display_name = 'My very own cluster' - cluster.create() - -If you would like more than the minimum number of nodes (``3``) in your cluster: - -.. code:: python - - cluster.serve_nodes = 10 - cluster.create() - -Check on Current Operation --------------------------- - -.. note:: - - When modifying a cluster (via a `CreateCluster`_, `UpdateCluster`_ or - `UndeleteCluster`_ request), the Bigtable API will return a long-running - `Operation`_. This will be stored on the object after each of - :meth:`create() `, - :meth:`update() ` and - :meth:`undelete() ` are called. - -.. _Operation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L73-L102 - -You can check if a long-running operation (for a -:meth:`create() `, -:meth:`update() ` or -:meth:`undelete() `) has finished -by making a `GetOperation`_ request with -:meth:`operation_finished() `: - -.. code:: python - - >>> cluster.operation_finished() - True - -.. note:: - - The operation data is stored in protected fields on the - :class:`Cluster `: - ``_operation_type``, ``_operation_id`` and ``_operation_begin``. - If these are unset, then - :meth:`operation_finished() ` - will fail. Also, these will be removed after a long-running operation - has completed (checked via this method). We could easily surface these - properties publicly, but it's unclear if end-users would need them. - -Get metadata for an existing Cluster ------------------------------------- - -After creating the cluster object, make a `GetCluster`_ API request -with :meth:`reload() `: - -.. code:: python - - cluster.reload() - -This will load ``serve_nodes`` and ``display_name`` for the existing -``cluster`` in addition to the ``cluster_id``, ``zone`` and ``project`` -already set on the :class:`Cluster ` object. - -Update an existing Cluster --------------------------- - -After creating the cluster object, make an `UpdateCluster`_ API request -with :meth:`update() `: - -.. code:: python - - client.display_name = 'New display_name' - cluster.update() - -Delete an existing Cluster --------------------------- - -Make a `DeleteCluster`_ API request with -:meth:`delete() `: - -.. code:: python - - cluster.delete() - -Undelete a deleted Cluster --------------------------- - -Make an `UndeleteCluster`_ API request with -:meth:`undelete() `: - -.. code:: python - - cluster.undelete() - -Next Step ---------- - -Now we go down the hierarchy from -:class:`Cluster ` to a -:class:`Table `. - -Head next to learn about the `Table Admin API`_. - -.. _Cluster Admin API: https://cloud.google.com/bigtable/docs/creating-cluster -.. _CreateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L66-L68 -.. _GetCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L38-L40 -.. _UpdateCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L93-L95 -.. _DeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L109-L111 -.. _ListZones: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L33-L35 -.. _ListClusters: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L44-L46 -.. _GetOperation: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/bfe4138f04bf3383a558152e4333112cdd13d5b0/bigtable-protos/src/main/proto/google/longrunning/operations.proto#L43-L45 -.. _UndeleteCluster: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/e6fc386d9adc821e1cf5c175c5bf5830b641eb3f/bigtable-protos/src/main/proto/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto#L126-L128 -.. _Table Admin API: table-api.html diff --git a/docs/cluster.rst b/docs/cluster.rst deleted file mode 100644 index d22d213..0000000 --- a/docs/cluster.rst +++ /dev/null @@ -1,7 +0,0 @@ -Cluster -~~~~~~~ - -.. automodule:: gcloud_bigtable.cluster - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/column-family.rst b/docs/column-family.rst deleted file mode 100644 index 1210803..0000000 --- a/docs/column-family.rst +++ /dev/null @@ -1,48 +0,0 @@ -Column Families -=============== - -When creating a -:class:`ColumnFamily `, it is -possible to set garbage collection rules for expired data. - -By setting a rule, cells in the table matching the rule will be deleted -during periodic garbage collection (which executes opportunistically in the -background). - -The types -:class:`GarbageCollectionRule `, -:class:`GarbageCollectionRuleUnion ` and -:class:`GarbageCollectionRuleIntersection ` -can all be used as the optional ``gc_rule`` argument in the -:class:`ColumnFamily ` -constructor. This value is then used in the -:meth:`create() ` and -:meth:`update() ` methods. - -These rules can be nested arbitrarily, with -:class:`GarbageCollectionRule ` -at the lowest level of the nesting: - -.. code:: python - - import datetime - - max_age = datetime.timedelta(days=3) - rule1 = GarbageCollectionRule(max_age=max_age) - rule2 = GarbageCollectionRule(max_num_versions=1) - - # Make a composite that matches anything older than 3 days **AND** - # with more than 1 version. - rule3 = GarbageCollectionIntersection(rules=[rule1, rule2]) - - # Make another composite that matches our previous intersection - # **OR** anything that has more than 3 versions. - rule4 = GarbageCollectionRule(max_num_versions=3) - rule5 = GarbageCollectionUnion(rules=[rule3, rule4]) - ----- - -.. automodule:: gcloud_bigtable.column_family - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/conf.py b/docs/conf.py index 60aaa12..515e0a5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,6 @@ from email import message_from_string import os -from pkg_resources import get_distribution import sys import types @@ -68,8 +67,7 @@ # The short X.Y version. # version = '0.0.1' # The full version, including alpha/beta/rc tags. -distro = get_distribution('gcloud_bigtable') -release = os.getenv('SPHINX_RELEASE', distro.version) +release = os.getenv('SPHINX_RELEASE', '0.0.retired') # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -263,8 +261,7 @@ def add_grpc_mock(grpc_mod, subpackage, module_names): # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). -metadata = distro.get_metadata(distro.PKG_INFO) -author = message_from_string(metadata).get('Author') +author = 'Google Cloud Platform' latex_documents = [ (master_doc, 'GoogleCloudBigtable.tex', u'Google Cloud Bigtable Documentation', author, 'manual'), diff --git a/docs/data-api.rst b/docs/data-api.rst deleted file mode 100644 index fc41ad3..0000000 --- a/docs/data-api.rst +++ /dev/null @@ -1,327 +0,0 @@ -Data API -======== - -After creating a :class:`Table ` and some -column families, you are ready to store and retrieve data. - -Cells vs. Columns vs. Column Families -+++++++++++++++++++++++++++++++++++++ - -* As we saw before, a table can have many column families. -* As we'll see below, a table also has many rows (specified by row keys). -* Within a row, data is stored in a cell. A cell simply has a value (as - bytes) and a timestamp. The number of cells in each row can be - different, depending on what was stored in each row. -* Each cell lies in a column (**not** a column family). A column is really - just a more **specific** modifier within a column family. A column - can be present in every way, in only one or anywhere in between. -* Within a column family there can be many columns. For example within - the column family ``foo`` we could have columns ``bar`` and ``baz``. - These would typically be represented as ``foo:bar`` and ``foo:baz``. - -Modifying Data -++++++++++++++ - -Since data is stored in cells, which are stored in rows, the -:class:`Row ` class is the only class used to -modify (write, update, delete) data in a -:class:`Table `. - -Row Factory ------------ - -To create a :class:`Row ` object - -.. code:: python - - row = table.row(row_key) - -Unlike the previous string values we've used before, the row key must -be ``bytes``. - -Direct vs. Conditional vs. Append ---------------------------------- - -There are three ways to modify data in a table, described by the -`MutateRow`_, `CheckAndMutateRow`_ and `ReadModifyWriteRow`_ API -methods. - -* The **direct** way is via `MutateRow`_ which involves simply - adding, overwriting or deleting cells. -* The **conditional** way is via `CheckAndMutateRow`_. This method - first checks if some filter is matched in a a given row, then - applies one of two sets of mutations, depending on if a match - occurred or not. -* The **append** way is via `ReadModifyWriteRow`_. This simply - appends (as bytes) or increments (as an integer) data in a presumed - existing cell in a row. - -Building Up Mutations ---------------------- - -In all three cases, a set of mutations (or two sets) are built up -on a :class:`Row ` before they are sent of -in a batch via :meth:`commit() `: - -.. code:: python - - row.commit() - -To send **append** mutations in batch, use -:meth:`commit_modifications() `: - -.. code:: python - - row.commit_modifications() - -We have a small set of methods on the :class:`Row ` -to build these mutations up. - -Direct Mutations ----------------- - -Direct mutations can be added via one of four methods - -* :meth:`set_cell() ` allows a - single value to be written to a column - - .. code:: python - - row.set_cell(column_family_id, column, value, - timestamp=timestamp) - - If the ``timestamp`` is omitted, the current time on the Google Cloud - Bigtable server will be used when the cell is stored. - - The value can either by bytes or an integer (which will be converted to - bytes as an unsigned 64-bit integer). - -* :meth:`delete_cell() ` deletes - all cells (i.e. for all timestamps) in a given column - - .. code:: python - - row.delete_cell(column_family_id, column) - - Remember, this only happens in the ``row`` we are using. - - If we only want to delete cells from a limited range of time, a - :class:`TimestampRange ` can - be used - - .. code:: python - - row.delete_cell(column_family_id, column, - time_range=time_range) - -* :meth:`delete_cells() ` does - the same thing as :meth:`delete_cell() ` - but accepts a list of columns in a column family rather than a single one. - - .. code:: python - - row.delete_cells(column_family_id, [column1, column2], - time_range=time_range) - - In addition, if we want to delete cells from every column in a column family, - the special :attr:`ALL_COLUMNS ` value - can be used - - .. code:: python - - row.delete_cells(column_family_id, Row.ALL_COLUMNS, - time_range=time_range) - -* :meth:`delete() ` will delete the entire row - - .. code:: python - - row.delete() - -Conditional Mutations ---------------------- - -Making **conditional** conditional modifications is essentially identical -to **direct** modifications, but we need to specify a filter to match -against in the row: - -.. code:: python - - row = table.row(row_key, filter_=filter_val) - -See the :class:`Row ` class for more information -about acceptable values for ``filter_``. - -The only other difference from **direct** modifications are that each mutation -added must specify a ``state``: will the mutation be applied if the filter -matches or if it fails to match. - -For example - -.. code:: python - - row.set_cell(column_family_id, column, value, - timestamp=timestamp, state=True) - -.. note:: - - If ``state`` is passed when no ``filter_`` is set on a - :class:`Row `, adding the mutation will fail. - Similarly, if no ``state`` is passed when a ``filter_`` has been set, - adding the mutation will fail. - -Append Mutations ----------------- - -Append mutations can be added via one of two methods - -* :meth:`append_cell_value ` appends - a bytes value to an existing cell: - - .. code:: python - - row.append_cell_value(column_family_id, column, bytes_value) - -* :meth:`increment_cell_value ` increments - an integer value in an existing cell: - - .. code:: python - - row.increment_cell_value(column_family_id, column, int_value) - - Since only bytes are stored in a cell, the current value is decoded as - an unsigned 64-bit integer before being incremented. (This happens on - the Google Cloud Bigtable server, not in the library.) - -Notice that no timestamp was specified. This is because **append** mutations -operate on the latest value of the specified column. - -If there are no cells in the specified column, then the empty string (bytes -case) or zero (integer case) are the assumed values. - -Starting Fresh --------------- - -If accumulated mutations need to be dropped, use -:meth:`clear_mutations() ` - -.. code:: python - - row.clear_mutations() - -To clear **append** mutations, use -:meth:`clear_modification_rules() ` - -.. code:: python - - row.clear_modification_rules() - -Reading Data -++++++++++++ - -Read Single Row from a Table ----------------------------- - -To make a `ReadRows`_ API request for a single row key, use -:meth:`Table.read_row() `: - -.. code:: python - - row_data = table.read_row(row_key) - -Rather than returning a :class:`Row `, this method -returns a :class:`PartialRowData ` -instance. This class is used for reading and parsing data rather than for -modifying data (as :class:`Row ` is). - -A filter can also be applied to the - -.. code:: python - - row_data = table.read_row(row_key, filter_=filter_val) - -The allowable ``filter_`` values are the same as those used for a -:class:`Row ` with **conditional** mutations. For -more information, see the -:meth:`Table.read_row() ` documentation. - -Stream Many Rows from a Table ------------------------------ - -To make a `ReadRows`_ API request for a stream of rows, use -:meth:`Table.read_rows() `: - -.. code:: python - - row_data = table.read_rows() - -Using gRPC over HTTP/2, a continual stream of responses will be delivered. -We have a custom -returns a :class:`PartialRowsData ` -class to allow consuming and parsing these streams as they come. - -In particular - -* :meth:`consume_next() ` - pulls the next result from the stream, parses it and stores it on the - :class:`PartialRowsData ` instance -* :meth:`consume_all() ` - pulls results from the stream until there are no more -* :meth:`cancel() ` closes - the stream - -See the :class:`PartialRowsData ` -documentation for more information. - -As with -:meth:`Table.read_row() `, an optional -``filter_`` can be applied. In addition a ``start_key`` and / or ``end_key`` -can be supplied for the stream, a ``limit`` can be set and a boolean -``allow_row_interleaving`` can be specified to allow faster streamed results -at the potential cost of non-sequential reads. - -See the :meth:`Table.read_rows() ` -documentation for more information on the optional arguments. - -Sample Keys in a Table ----------------------- - -Make a `SampleRowKeys`_ API request with -:meth:`Table.sample_row_keys() `: - -.. code:: python - - keys_iterator = table.sample_row_keys() - -The returned row keys will delimit contiguous sections of the table of -approximately equal size, which can be used to break up the data for -distributed tasks like mapreduces. - -As with -:meth:`Table.read_rows() `, the -returned ``keys_iterator`` is connected to a cancellable HTTP/2 stream. - -The next key in the result can be accessed via - -.. code:: python - - next_key = keys_iterator.next() - -or all keys can be iterated over via - -.. code:: python - - for curr_key in keys_iterator: - do_something(curr_key) - -Just as with reading, the stream can be canceled: - -.. code:: python - - keys_iterator.cancel() - -.. _ReadRows: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L36-L38 -.. _SampleRowKeys: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L44-L46 -.. _MutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L50-L52 -.. _CheckAndMutateRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L55-L57 -.. _ReadModifyWriteRow: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/v1/bigtable_service.proto#L63-L65 diff --git a/docs/happybase-batch.rst b/docs/happybase-batch.rst deleted file mode 100644 index 5c386e8..0000000 --- a/docs/happybase-batch.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Batch -~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.batch - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-connection.rst b/docs/happybase-connection.rst deleted file mode 100644 index 7d60940..0000000 --- a/docs/happybase-connection.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Connection -~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.connection - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-package.rst b/docs/happybase-package.rst deleted file mode 100644 index 717d549..0000000 --- a/docs/happybase-package.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Package -~~~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.__init__ - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-pool.rst b/docs/happybase-pool.rst deleted file mode 100644 index 0b36ae1..0000000 --- a/docs/happybase-pool.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Connection Pool -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.pool - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/happybase-table.rst b/docs/happybase-table.rst deleted file mode 100644 index 0809e52..0000000 --- a/docs/happybase-table.rst +++ /dev/null @@ -1,7 +0,0 @@ -HappyBase Table -~~~~~~~~~~~~~~~ - -.. automodule:: gcloud_bigtable.happybase.table - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index 77da93f..6c72fe8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,82 +1,8 @@ -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Getting Started - - client-intro - client - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Cluster Admin - - cluster-api - cluster - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Table Admin - - table-api - table - column-family - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: Data API - - data-api - row - row-data - -.. toctree:: - :maxdepth: 2 - :hidden: - :caption: HappyBase Compatibility Sub-Package - - happybase-package - happybase-table - happybase-connection - happybase-batch - happybase-pool - Google Cloud Bigtable: Python ============================= -This library is an alpha implementation of `Google Cloud Bigtable`_ -and is closely related to `gcloud-python`_. - -API requests are sent to the Google Cloud Bigtable API via RPC over HTTP/2. -In order to support this, we'll rely on `gRPC`_. We are working with the gRPC -team to rapidly make the install story more user-friendly. - -Get started by learning about the -:class:`Client ` on the `Base for Everything`_ -page. If you have install questions, check out the project's `README`_. - -In the hierarchy of API concepts - -* a :class:`Client ` owns a - :class:`Cluster ` -* a :class:`Cluster ` owns a - :class:`Table ` -* a :class:`Table ` owns a - :class:`ColumnFamily ` -* a :class:`Table ` owns a - :class:`Row ` - (and all the cells in the row) - -Indices and tables -~~~~~~~~~~~~~~~~~~ - -* :ref:`genindex` -* :ref:`modindex` +This library was an alpha implementation of `Google Cloud Bigtable`_. +The project has been moved into `gcloud-python`_. .. _Google Cloud Bigtable: https://cloud.google.com/bigtable/docs/ -.. _gcloud-python: http://gcloud-python.readthedocs.org/en/latest/ -.. _gRPC: http://www.grpc.io/ -.. _README: https://github.com/dhermes/gcloud-python-bigtable/blob/master/README.md -.. _Base for Everything: client-intro.html +.. _gcloud-python: http://gcloud-python.readthedocs.io/en/stable/bigtable-usage.html diff --git a/docs/row-data.rst b/docs/row-data.rst deleted file mode 100644 index 7a4c235..0000000 --- a/docs/row-data.rst +++ /dev/null @@ -1,7 +0,0 @@ -Row Data -~~~~~~~~ - -.. automodule:: gcloud_bigtable.row_data - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/row.rst b/docs/row.rst deleted file mode 100644 index 4603702..0000000 --- a/docs/row.rst +++ /dev/null @@ -1,7 +0,0 @@ -Row -~~~ - -.. automodule:: gcloud_bigtable.row - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/table-api.rst b/docs/table-api.rst deleted file mode 100644 index f90719c..0000000 --- a/docs/table-api.rst +++ /dev/null @@ -1,172 +0,0 @@ -Table Admin API -=============== - -After creating a :class:`Cluster `, you can -interact with individual tables, groups of tables or column families within -a table. - -List Tables ------------ - -If you want a comprehensive list of all existing tables in a cluster, make a -`ListTables`_ API request with -:meth:`Cluster.list_tables() `: - -.. code:: python - - tables = cluster.list_tables() - -Table Factory -------------- - -To create a :class:`Table ` object: - -.. code:: python - - table = cluster.table(table_id) - -Even if this :class:`Table ` already -has been created with the API, you'll want this object to use as a -parent of a :class:`ColumnFamily ` -or :class:`Row `. - -Create a new Table ------------------- - -After creating the table object, make a `CreateTable`_ API request -with :meth:`create() `: - -.. code:: python - - table.create() - -If you would to initially split the table into several tablets (Tablets are -similar to HBase regions): - -.. code:: python - - table.create(initial_split_keys=['s1', 's2']) - -Delete an existing Table ------------------------- - -Make a `DeleteTable`_ API request with -:meth:`delete() `: - -.. code:: python - - table.delete() - -Rename an existing Table ------------------------- - -Though the `RenameTable`_ API request is listed in the service -definition, requests to that method return:: - - BigtableTableService.RenameTable is not yet implemented - -We have implemented :meth:`rename() ` -but it will not work unless the backend supports the method. - -List Column Families in a Table -------------------------------- - -Though there is no **official** method for retrieving `column families`_ -associated with a table, the `GetTable`_ API method returns a -table object with the names of the column families. - -To retrieve the list of column families use -:meth:`list_column_families() `: - -.. code:: python - - column_families = table.list_column_families() - -.. note:: - - Unfortunately the garbage collection rules used to create each column family - are not returned in the `GetTable`_ response. - -Column Family Factory ---------------------- - -To create a -:class:`ColumnFamily ` object: - -.. code:: python - - column_family = table.column_family(column_family_id) - -There is no real reason to use this factory unless you intend to -create or delete a column family. - -In addition, you can specify an optional ``gc_rule`` (a -:class:`GarbageCollectionRule ` -or similar): - -.. code:: python - - column_family = table.column_family(column_family_id, - gc_rule=gc_rule) - -This rule helps the backend determine when and how to clean up old cells -in the column family. - -See the `Column Families doc`_ for more information about -:class:`GarbageCollectionRule ` -and related classes. - -Create a new Column Family --------------------------- - -After creating the column family object, make a `CreateColumnFamily`_ API -request with -:meth:`ColumnFamily.create() ` - -.. code:: python - - column_family.create() - -Delete an existing Column Family --------------------------------- - -Make a `DeleteColumnFamily`_ API request with -:meth:`ColumnFamily.delete() ` - -.. code:: python - - column_family.delete() - -Update an existing Column Family --------------------------------- - -Though the `UpdateColumnFamily`_ API request is listed in the service -definition, requests to that method return:: - - BigtableTableService.UpdateColumnFamily is not yet implemented - -We have implemented -:meth:`ColumnFamily.update() ` -but it will not work unless the backend supports the method. - -Next Step ---------- - -Now we go down the final step of the hierarchy from -:class:`Table ` to -:class:`Row ` as well as streaming -data directly via a :class:`Table `. - -Head next to learn about the `Data API`_. - -.. _ListTables: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L40-L42 -.. _CreateTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L35-L37 -.. _DeleteTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L50-L52 -.. _RenameTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L58-L58 -.. _GetTable: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L45-L47 -.. _CreateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L61-L63 -.. _UpdateColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L66-L68 -.. _DeleteColumnFamily: https://github.com/GoogleCloudPlatform/cloud-bigtable-client/blob/f4d922bb950f1584b30f9928e84d042ad59f5658/bigtable-protos/src/main/proto/google/bigtable/admin/table/v1/bigtable_table_service.proto#L71-L73 -.. _column families: https://cloud.google.com/bigtable/docs/schema-design#column_families_and_column_qualifiers -.. _Column Families doc: column-family.html -.. _Data API: data-api.html diff --git a/docs/table.rst b/docs/table.rst deleted file mode 100644 index affb0f1..0000000 --- a/docs/table.rst +++ /dev/null @@ -1,7 +0,0 @@ -Table -~~~~~ - -.. automodule:: gcloud_bigtable.table - :members: - :undoc-members: - :show-inheritance: diff --git a/gcloud_bigtable/__init__.py b/gcloud_bigtable/__init__.py deleted file mode 100644 index 251590d..0000000 --- a/gcloud_bigtable/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable API package.""" - -try: - from grpc._adapter import _c -except ImportError as exc: - if 'libgrpc.so' in str(exc): - raise ImportError('gRPC libraries could not be located. Please see ' - 'instructions to locate these files. You\'ll want ' - 'to set your LD_LIBRARY_PATH variable to help ' - 'Python locate the libraries.') - else: - raise diff --git a/gcloud_bigtable/_generated/__init__.py b/gcloud_bigtable/_generated/__init__.py deleted file mode 100644 index ad35adc..0000000 --- a/gcloud_bigtable/_generated/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Generated protobuf modules for Google Cloud Bigtable API.""" diff --git a/gcloud_bigtable/_generated/annotations_pb2.py b/gcloud_bigtable/_generated/annotations_pb2.py deleted file mode 100644 index 64545bc..0000000 --- a/gcloud_bigtable/_generated/annotations_pb2.py +++ /dev/null @@ -1,49 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/api/annotations.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import http_pb2 as google_dot_api_dot_http__pb2 -from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/api/annotations.proto', - package='google.api', - syntax='proto3', - serialized_pb=_b('\n\x1cgoogle/api/annotations.proto\x12\ngoogle.api\x1a\x15google/api/http.proto\x1a google/protobuf/descriptor.proto:E\n\x04http\x12\x1e.google.protobuf.MethodOptions\x18\xb0\xca\xbc\" \x01(\x0b\x32\x14.google.api.HttpRuleB$\n\x0e\x63om.google.apiB\x10\x41nnotationsProtoP\x01\x62\x06proto3') - , - dependencies=[google_dot_api_dot_http__pb2.DESCRIPTOR,google_dot_protobuf_dot_descriptor__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - -HTTP_FIELD_NUMBER = 72295728 -http = _descriptor.FieldDescriptor( - name='http', full_name='google.api.http', index=0, - number=72295728, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) - -DESCRIPTOR.extensions_by_name['http'] = http - -http.message_type = google_dot_api_dot_http__pb2._HTTPRULE -google_dot_protobuf_dot_descriptor__pb2.MethodOptions.RegisterExtension(http) - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\016com.google.apiB\020AnnotationsProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/any_pb2.py b/gcloud_bigtable/_generated/any_pb2.py deleted file mode 100644 index 4c31a15..0000000 --- a/gcloud_bigtable/_generated/any_pb2.py +++ /dev/null @@ -1,81 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/any.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/any.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=_b('\n\x19google/protobuf/any.proto\x12\x0fgoogle.protobuf\"&\n\x03\x41ny\x12\x10\n\x08type_url\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x0c\x42$\n\x13\x63om.google.protobufB\x08\x41nyProtoP\x01\xa0\x01\x01\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_ANY = _descriptor.Descriptor( - name='Any', - full_name='google.protobuf.Any', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='type_url', full_name='google.protobuf.Any.type_url', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.protobuf.Any.value', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=46, - serialized_end=84, -) - -DESCRIPTOR.message_types_by_name['Any'] = _ANY - -Any = _reflection.GeneratedProtocolMessageType('Any', (_message.Message,), dict( - DESCRIPTOR = _ANY, - __module__ = 'google.protobuf.any_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Any) - )) -_sym_db.RegisterMessage(Any) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\010AnyProtoP\001\240\001\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py b/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py deleted file mode 100644 index 544af89..0000000 --- a/gcloud_bigtable/_generated/bigtable_cluster_data_pb2.py +++ /dev/null @@ -1,229 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from gcloud_bigtable._generated import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/cluster/v1/bigtable_cluster_data.proto', - package='google.bigtable.admin.cluster.v1', - syntax='proto3', - serialized_pb=_b('\n""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ListZones(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def GetCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListClusters(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CreateCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UpdateCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteCluster(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UndeleteCluster(self, request, context): - raise NotImplementedError() -class EarlyAdopterBigtableClusterServiceServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterBigtableClusterServiceStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ListZones(self, request): - raise NotImplementedError() - ListZones.async = None - @abc.abstractmethod - def GetCluster(self, request): - raise NotImplementedError() - GetCluster.async = None - @abc.abstractmethod - def ListClusters(self, request): - raise NotImplementedError() - ListClusters.async = None - @abc.abstractmethod - def CreateCluster(self, request): - raise NotImplementedError() - CreateCluster.async = None - @abc.abstractmethod - def UpdateCluster(self, request): - raise NotImplementedError() - UpdateCluster.async = None - @abc.abstractmethod - def DeleteCluster(self, request): - raise NotImplementedError() - DeleteCluster.async = None - @abc.abstractmethod - def UndeleteCluster(self, request): - raise NotImplementedError() - UndeleteCluster.async = None -def early_adopter_create_BigtableClusterService_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.operations_pb2 - method_service_descriptions = { - "CreateCluster": utilities.unary_unary_service_description( - servicer.CreateCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ), - "DeleteCluster": utilities.unary_unary_service_description( - servicer.DeleteCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "GetCluster": utilities.unary_unary_service_description( - servicer.GetCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ), - "ListClusters": utilities.unary_unary_service_description( - servicer.ListClusters, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.SerializeToString, - ), - "ListZones": utilities.unary_unary_service_description( - servicer.ListZones, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.FromString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.SerializeToString, - ), - "UndeleteCluster": utilities.unary_unary_service_description( - servicer.UndeleteCluster, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.FromString, - gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, - ), - "UpdateCluster": utilities.unary_unary_service_description( - servicer.UpdateCluster, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - ), - } - return implementations.server("google.bigtable.admin.cluster.v1.BigtableClusterService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_BigtableClusterService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_data_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2 - import gcloud_bigtable._generated.operations_pb2 - method_invocation_descriptions = { - "CreateCluster": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.CreateClusterRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ), - "DeleteCluster": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.DeleteClusterRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "GetCluster": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.GetClusterRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ), - "ListClusters": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListClustersResponse.FromString, - ), - "ListZones": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.ListZonesResponse.FromString, - ), - "UndeleteCluster": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_service_messages_pb2.UndeleteClusterRequest.SerializeToString, - gcloud_bigtable._generated.operations_pb2.Operation.FromString, - ), - "UpdateCluster": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.SerializeToString, - gcloud_bigtable._generated.bigtable_cluster_data_pb2.Cluster.FromString, - ), - } - return implementations.stub("google.bigtable.admin.cluster.v1.BigtableClusterService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_data_pb2.py b/gcloud_bigtable/_generated/bigtable_data_pb2.py deleted file mode 100644 index cb8f0b1..0000000 --- a/gcloud_bigtable/_generated/bigtable_data_pb2.py +++ /dev/null @@ -1,1184 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/v1/bigtable_data.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/v1/bigtable_data.proto', - package='google.bigtable.v1', - syntax='proto3', - serialized_pb=_b('\n&google/bigtable/v1/bigtable_data.proto\x12\x12google.bigtable.v1\"@\n\x03Row\x12\x0b\n\x03key\x18\x01 \x01(\x0c\x12,\n\x08\x66\x61milies\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Family\"C\n\x06\x46\x61mily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12+\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x1a.google.bigtable.v1.Column\"D\n\x06\x43olumn\x12\x11\n\tqualifier\x18\x01 \x01(\x0c\x12\'\n\x05\x63\x65lls\x18\x02 \x03(\x0b\x32\x18.google.bigtable.v1.Cell\"?\n\x04\x43\x65ll\x12\x18\n\x10timestamp_micros\x18\x01 \x01(\x03\x12\r\n\x05value\x18\x02 \x01(\x0c\x12\x0e\n\x06labels\x18\x03 \x03(\t\".\n\x08RowRange\x12\x11\n\tstart_key\x18\x02 \x01(\x0c\x12\x0f\n\x07\x65nd_key\x18\x03 \x01(\x0c\"\xd6\x01\n\x0b\x43olumnRange\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12#\n\x19start_qualifier_inclusive\x18\x02 \x01(\x0cH\x00\x12#\n\x19start_qualifier_exclusive\x18\x03 \x01(\x0cH\x00\x12!\n\x17\x65nd_qualifier_inclusive\x18\x04 \x01(\x0cH\x01\x12!\n\x17\x65nd_qualifier_exclusive\x18\x05 \x01(\x0cH\x01\x42\x11\n\x0fstart_qualifierB\x0f\n\rend_qualifier\"N\n\x0eTimestampRange\x12\x1e\n\x16start_timestamp_micros\x18\x01 \x01(\x03\x12\x1c\n\x14\x65nd_timestamp_micros\x18\x02 \x01(\x03\"\xa8\x01\n\nValueRange\x12\x1f\n\x15start_value_inclusive\x18\x01 \x01(\x0cH\x00\x12\x1f\n\x15start_value_exclusive\x18\x02 \x01(\x0cH\x00\x12\x1d\n\x13\x65nd_value_inclusive\x18\x03 \x01(\x0cH\x01\x12\x1d\n\x13\x65nd_value_exclusive\x18\x04 \x01(\x0cH\x01\x42\r\n\x0bstart_valueB\x0b\n\tend_value\"\xdf\x08\n\tRowFilter\x12\x34\n\x05\x63hain\x18\x01 \x01(\x0b\x32#.google.bigtable.v1.RowFilter.ChainH\x00\x12>\n\ninterleave\x18\x02 \x01(\x0b\x32(.google.bigtable.v1.RowFilter.InterleaveH\x00\x12<\n\tcondition\x18\x03 \x01(\x0b\x32\'.google.bigtable.v1.RowFilter.ConditionH\x00\x12\x0e\n\x04sink\x18\x10 \x01(\x08H\x00\x12\x19\n\x0fpass_all_filter\x18\x11 \x01(\x08H\x00\x12\x1a\n\x10\x62lock_all_filter\x18\x12 \x01(\x08H\x00\x12\x1e\n\x14row_key_regex_filter\x18\x04 \x01(\x0cH\x00\x12\x1b\n\x11row_sample_filter\x18\x0e \x01(\x01H\x00\x12\"\n\x18\x66\x61mily_name_regex_filter\x18\x05 \x01(\tH\x00\x12\'\n\x1d\x63olumn_qualifier_regex_filter\x18\x06 \x01(\x0cH\x00\x12>\n\x13\x63olumn_range_filter\x18\x07 \x01(\x0b\x32\x1f.google.bigtable.v1.ColumnRangeH\x00\x12\x44\n\x16timestamp_range_filter\x18\x08 \x01(\x0b\x32\".google.bigtable.v1.TimestampRangeH\x00\x12\x1c\n\x12value_regex_filter\x18\t \x01(\x0cH\x00\x12<\n\x12value_range_filter\x18\x0f \x01(\x0b\x32\x1e.google.bigtable.v1.ValueRangeH\x00\x12%\n\x1b\x63\x65lls_per_row_offset_filter\x18\n \x01(\x05H\x00\x12$\n\x1a\x63\x65lls_per_row_limit_filter\x18\x0b \x01(\x05H\x00\x12\'\n\x1d\x63\x65lls_per_column_limit_filter\x18\x0c \x01(\x05H\x00\x12!\n\x17strip_value_transformer\x18\r \x01(\x08H\x00\x12!\n\x17\x61pply_label_transformer\x18\x13 \x01(\tH\x00\x1a\x37\n\x05\x43hain\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a<\n\nInterleave\x12.\n\x07\x66ilters\x18\x01 \x03(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x1a\xad\x01\n\tCondition\x12\x37\n\x10predicate_filter\x18\x01 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x32\n\x0btrue_filter\x18\x02 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x33\n\x0c\x66\x61lse_filter\x18\x03 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilterB\x08\n\x06\x66ilter\"\xc9\x04\n\x08Mutation\x12\x38\n\x08set_cell\x18\x01 \x01(\x0b\x32$.google.bigtable.v1.Mutation.SetCellH\x00\x12K\n\x12\x64\x65lete_from_column\x18\x02 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromColumnH\x00\x12K\n\x12\x64\x65lete_from_family\x18\x03 \x01(\x0b\x32-.google.bigtable.v1.Mutation.DeleteFromFamilyH\x00\x12\x45\n\x0f\x64\x65lete_from_row\x18\x04 \x01(\x0b\x32*.google.bigtable.v1.Mutation.DeleteFromRowH\x00\x1a\x61\n\x07SetCell\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x18\n\x10timestamp_micros\x18\x03 \x01(\x03\x12\r\n\x05value\x18\x04 \x01(\x0c\x1ay\n\x10\x44\x65leteFromColumn\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x36\n\ntime_range\x18\x03 \x01(\x0b\x32\".google.bigtable.v1.TimestampRange\x1a\'\n\x10\x44\x65leteFromFamily\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x1a\x0f\n\rDeleteFromRowB\n\n\x08mutation\"\x80\x01\n\x13ReadModifyWriteRule\x12\x13\n\x0b\x66\x61mily_name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_qualifier\x18\x02 \x01(\x0c\x12\x16\n\x0c\x61ppend_value\x18\x03 \x01(\x0cH\x00\x12\x1a\n\x10increment_amount\x18\x04 \x01(\x03H\x00\x42\x06\n\x04ruleB-\n\x16\x63om.google.bigtable.v1B\x11\x42igtableDataProtoP\x01\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_ROW = _descriptor.Descriptor( - name='Row', - full_name='google.bigtable.v1.Row', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='google.bigtable.v1.Row.key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='families', full_name='google.bigtable.v1.Row.families', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=62, - serialized_end=126, -) - - -_FAMILY = _descriptor.Descriptor( - name='Family', - full_name='google.bigtable.v1.Family', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.v1.Family.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='columns', full_name='google.bigtable.v1.Family.columns', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=128, - serialized_end=195, -) - - -_COLUMN = _descriptor.Descriptor( - name='Column', - full_name='google.bigtable.v1.Column', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='qualifier', full_name='google.bigtable.v1.Column.qualifier', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells', full_name='google.bigtable.v1.Column.cells', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=197, - serialized_end=265, -) - - -_CELL = _descriptor.Descriptor( - name='Cell', - full_name='google.bigtable.v1.Cell', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='timestamp_micros', full_name='google.bigtable.v1.Cell.timestamp_micros', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.bigtable.v1.Cell.value', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='labels', full_name='google.bigtable.v1.Cell.labels', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=267, - serialized_end=330, -) - - -_ROWRANGE = _descriptor.Descriptor( - name='RowRange', - full_name='google.bigtable.v1.RowRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start_key', full_name='google.bigtable.v1.RowRange.start_key', index=0, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_key', full_name='google.bigtable.v1.RowRange.end_key', index=1, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=332, - serialized_end=378, -) - - -_COLUMNRANGE = _descriptor.Descriptor( - name='ColumnRange', - full_name='google.bigtable.v1.ColumnRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.ColumnRange.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='start_qualifier_inclusive', full_name='google.bigtable.v1.ColumnRange.start_qualifier_inclusive', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='start_qualifier_exclusive', full_name='google.bigtable.v1.ColumnRange.start_qualifier_exclusive', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_qualifier_inclusive', full_name='google.bigtable.v1.ColumnRange.end_qualifier_inclusive', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_qualifier_exclusive', full_name='google.bigtable.v1.ColumnRange.end_qualifier_exclusive', index=4, - number=5, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='start_qualifier', full_name='google.bigtable.v1.ColumnRange.start_qualifier', - index=0, containing_type=None, fields=[]), - _descriptor.OneofDescriptor( - name='end_qualifier', full_name='google.bigtable.v1.ColumnRange.end_qualifier', - index=1, containing_type=None, fields=[]), - ], - serialized_start=381, - serialized_end=595, -) - - -_TIMESTAMPRANGE = _descriptor.Descriptor( - name='TimestampRange', - full_name='google.bigtable.v1.TimestampRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start_timestamp_micros', full_name='google.bigtable.v1.TimestampRange.start_timestamp_micros', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_timestamp_micros', full_name='google.bigtable.v1.TimestampRange.end_timestamp_micros', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=597, - serialized_end=675, -) - - -_VALUERANGE = _descriptor.Descriptor( - name='ValueRange', - full_name='google.bigtable.v1.ValueRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start_value_inclusive', full_name='google.bigtable.v1.ValueRange.start_value_inclusive', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='start_value_exclusive', full_name='google.bigtable.v1.ValueRange.start_value_exclusive', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_value_inclusive', full_name='google.bigtable.v1.ValueRange.end_value_inclusive', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end_value_exclusive', full_name='google.bigtable.v1.ValueRange.end_value_exclusive', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='start_value', full_name='google.bigtable.v1.ValueRange.start_value', - index=0, containing_type=None, fields=[]), - _descriptor.OneofDescriptor( - name='end_value', full_name='google.bigtable.v1.ValueRange.end_value', - index=1, containing_type=None, fields=[]), - ], - serialized_start=678, - serialized_end=846, -) - - -_ROWFILTER_CHAIN = _descriptor.Descriptor( - name='Chain', - full_name='google.bigtable.v1.RowFilter.Chain', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='filters', full_name='google.bigtable.v1.RowFilter.Chain.filters', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1665, - serialized_end=1720, -) - -_ROWFILTER_INTERLEAVE = _descriptor.Descriptor( - name='Interleave', - full_name='google.bigtable.v1.RowFilter.Interleave', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='filters', full_name='google.bigtable.v1.RowFilter.Interleave.filters', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1722, - serialized_end=1782, -) - -_ROWFILTER_CONDITION = _descriptor.Descriptor( - name='Condition', - full_name='google.bigtable.v1.RowFilter.Condition', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='predicate_filter', full_name='google.bigtable.v1.RowFilter.Condition.predicate_filter', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='true_filter', full_name='google.bigtable.v1.RowFilter.Condition.true_filter', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='false_filter', full_name='google.bigtable.v1.RowFilter.Condition.false_filter', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1785, - serialized_end=1958, -) - -_ROWFILTER = _descriptor.Descriptor( - name='RowFilter', - full_name='google.bigtable.v1.RowFilter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='chain', full_name='google.bigtable.v1.RowFilter.chain', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='interleave', full_name='google.bigtable.v1.RowFilter.interleave', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='condition', full_name='google.bigtable.v1.RowFilter.condition', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='sink', full_name='google.bigtable.v1.RowFilter.sink', index=3, - number=16, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='pass_all_filter', full_name='google.bigtable.v1.RowFilter.pass_all_filter', index=4, - number=17, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='block_all_filter', full_name='google.bigtable.v1.RowFilter.block_all_filter', index=5, - number=18, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key_regex_filter', full_name='google.bigtable.v1.RowFilter.row_key_regex_filter', index=6, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_sample_filter', full_name='google.bigtable.v1.RowFilter.row_sample_filter', index=7, - number=14, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='family_name_regex_filter', full_name='google.bigtable.v1.RowFilter.family_name_regex_filter', index=8, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier_regex_filter', full_name='google.bigtable.v1.RowFilter.column_qualifier_regex_filter', index=9, - number=6, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_range_filter', full_name='google.bigtable.v1.RowFilter.column_range_filter', index=10, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='timestamp_range_filter', full_name='google.bigtable.v1.RowFilter.timestamp_range_filter', index=11, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value_regex_filter', full_name='google.bigtable.v1.RowFilter.value_regex_filter', index=12, - number=9, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value_range_filter', full_name='google.bigtable.v1.RowFilter.value_range_filter', index=13, - number=15, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells_per_row_offset_filter', full_name='google.bigtable.v1.RowFilter.cells_per_row_offset_filter', index=14, - number=10, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells_per_row_limit_filter', full_name='google.bigtable.v1.RowFilter.cells_per_row_limit_filter', index=15, - number=11, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cells_per_column_limit_filter', full_name='google.bigtable.v1.RowFilter.cells_per_column_limit_filter', index=16, - number=12, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='strip_value_transformer', full_name='google.bigtable.v1.RowFilter.strip_value_transformer', index=17, - number=13, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='apply_label_transformer', full_name='google.bigtable.v1.RowFilter.apply_label_transformer', index=18, - number=19, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_ROWFILTER_CHAIN, _ROWFILTER_INTERLEAVE, _ROWFILTER_CONDITION, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='filter', full_name='google.bigtable.v1.RowFilter.filter', - index=0, containing_type=None, fields=[]), - ], - serialized_start=849, - serialized_end=1968, -) - - -_MUTATION_SETCELL = _descriptor.Descriptor( - name='SetCell', - full_name='google.bigtable.v1.Mutation.SetCell', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.Mutation.SetCell.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier', full_name='google.bigtable.v1.Mutation.SetCell.column_qualifier', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='timestamp_micros', full_name='google.bigtable.v1.Mutation.SetCell.timestamp_micros', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.bigtable.v1.Mutation.SetCell.value', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2266, - serialized_end=2363, -) - -_MUTATION_DELETEFROMCOLUMN = _descriptor.Descriptor( - name='DeleteFromColumn', - full_name='google.bigtable.v1.Mutation.DeleteFromColumn', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.column_qualifier', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='time_range', full_name='google.bigtable.v1.Mutation.DeleteFromColumn.time_range', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2365, - serialized_end=2486, -) - -_MUTATION_DELETEFROMFAMILY = _descriptor.Descriptor( - name='DeleteFromFamily', - full_name='google.bigtable.v1.Mutation.DeleteFromFamily', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.Mutation.DeleteFromFamily.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2488, - serialized_end=2527, -) - -_MUTATION_DELETEFROMROW = _descriptor.Descriptor( - name='DeleteFromRow', - full_name='google.bigtable.v1.Mutation.DeleteFromRow', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2529, - serialized_end=2544, -) - -_MUTATION = _descriptor.Descriptor( - name='Mutation', - full_name='google.bigtable.v1.Mutation', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='set_cell', full_name='google.bigtable.v1.Mutation.set_cell', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete_from_column', full_name='google.bigtable.v1.Mutation.delete_from_column', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete_from_family', full_name='google.bigtable.v1.Mutation.delete_from_family', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete_from_row', full_name='google.bigtable.v1.Mutation.delete_from_row', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_MUTATION_SETCELL, _MUTATION_DELETEFROMCOLUMN, _MUTATION_DELETEFROMFAMILY, _MUTATION_DELETEFROMROW, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='mutation', full_name='google.bigtable.v1.Mutation.mutation', - index=0, containing_type=None, fields=[]), - ], - serialized_start=1971, - serialized_end=2556, -) - - -_READMODIFYWRITERULE = _descriptor.Descriptor( - name='ReadModifyWriteRule', - full_name='google.bigtable.v1.ReadModifyWriteRule', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='family_name', full_name='google.bigtable.v1.ReadModifyWriteRule.family_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_qualifier', full_name='google.bigtable.v1.ReadModifyWriteRule.column_qualifier', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='append_value', full_name='google.bigtable.v1.ReadModifyWriteRule.append_value', index=2, - number=3, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='increment_amount', full_name='google.bigtable.v1.ReadModifyWriteRule.increment_amount', index=3, - number=4, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='rule', full_name='google.bigtable.v1.ReadModifyWriteRule.rule', - index=0, containing_type=None, fields=[]), - ], - serialized_start=2559, - serialized_end=2687, -) - -_ROW.fields_by_name['families'].message_type = _FAMILY -_FAMILY.fields_by_name['columns'].message_type = _COLUMN -_COLUMN.fields_by_name['cells'].message_type = _CELL -_COLUMNRANGE.oneofs_by_name['start_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['start_qualifier_inclusive']) -_COLUMNRANGE.fields_by_name['start_qualifier_inclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['start_qualifier'] -_COLUMNRANGE.oneofs_by_name['start_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['start_qualifier_exclusive']) -_COLUMNRANGE.fields_by_name['start_qualifier_exclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['start_qualifier'] -_COLUMNRANGE.oneofs_by_name['end_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['end_qualifier_inclusive']) -_COLUMNRANGE.fields_by_name['end_qualifier_inclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['end_qualifier'] -_COLUMNRANGE.oneofs_by_name['end_qualifier'].fields.append( - _COLUMNRANGE.fields_by_name['end_qualifier_exclusive']) -_COLUMNRANGE.fields_by_name['end_qualifier_exclusive'].containing_oneof = _COLUMNRANGE.oneofs_by_name['end_qualifier'] -_VALUERANGE.oneofs_by_name['start_value'].fields.append( - _VALUERANGE.fields_by_name['start_value_inclusive']) -_VALUERANGE.fields_by_name['start_value_inclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['start_value'] -_VALUERANGE.oneofs_by_name['start_value'].fields.append( - _VALUERANGE.fields_by_name['start_value_exclusive']) -_VALUERANGE.fields_by_name['start_value_exclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['start_value'] -_VALUERANGE.oneofs_by_name['end_value'].fields.append( - _VALUERANGE.fields_by_name['end_value_inclusive']) -_VALUERANGE.fields_by_name['end_value_inclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['end_value'] -_VALUERANGE.oneofs_by_name['end_value'].fields.append( - _VALUERANGE.fields_by_name['end_value_exclusive']) -_VALUERANGE.fields_by_name['end_value_exclusive'].containing_oneof = _VALUERANGE.oneofs_by_name['end_value'] -_ROWFILTER_CHAIN.fields_by_name['filters'].message_type = _ROWFILTER -_ROWFILTER_CHAIN.containing_type = _ROWFILTER -_ROWFILTER_INTERLEAVE.fields_by_name['filters'].message_type = _ROWFILTER -_ROWFILTER_INTERLEAVE.containing_type = _ROWFILTER -_ROWFILTER_CONDITION.fields_by_name['predicate_filter'].message_type = _ROWFILTER -_ROWFILTER_CONDITION.fields_by_name['true_filter'].message_type = _ROWFILTER -_ROWFILTER_CONDITION.fields_by_name['false_filter'].message_type = _ROWFILTER -_ROWFILTER_CONDITION.containing_type = _ROWFILTER -_ROWFILTER.fields_by_name['chain'].message_type = _ROWFILTER_CHAIN -_ROWFILTER.fields_by_name['interleave'].message_type = _ROWFILTER_INTERLEAVE -_ROWFILTER.fields_by_name['condition'].message_type = _ROWFILTER_CONDITION -_ROWFILTER.fields_by_name['column_range_filter'].message_type = _COLUMNRANGE -_ROWFILTER.fields_by_name['timestamp_range_filter'].message_type = _TIMESTAMPRANGE -_ROWFILTER.fields_by_name['value_range_filter'].message_type = _VALUERANGE -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['chain']) -_ROWFILTER.fields_by_name['chain'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['interleave']) -_ROWFILTER.fields_by_name['interleave'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['condition']) -_ROWFILTER.fields_by_name['condition'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['sink']) -_ROWFILTER.fields_by_name['sink'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['pass_all_filter']) -_ROWFILTER.fields_by_name['pass_all_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['block_all_filter']) -_ROWFILTER.fields_by_name['block_all_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['row_key_regex_filter']) -_ROWFILTER.fields_by_name['row_key_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['row_sample_filter']) -_ROWFILTER.fields_by_name['row_sample_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['family_name_regex_filter']) -_ROWFILTER.fields_by_name['family_name_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['column_qualifier_regex_filter']) -_ROWFILTER.fields_by_name['column_qualifier_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['column_range_filter']) -_ROWFILTER.fields_by_name['column_range_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['timestamp_range_filter']) -_ROWFILTER.fields_by_name['timestamp_range_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['value_regex_filter']) -_ROWFILTER.fields_by_name['value_regex_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['value_range_filter']) -_ROWFILTER.fields_by_name['value_range_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['cells_per_row_offset_filter']) -_ROWFILTER.fields_by_name['cells_per_row_offset_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['cells_per_row_limit_filter']) -_ROWFILTER.fields_by_name['cells_per_row_limit_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['cells_per_column_limit_filter']) -_ROWFILTER.fields_by_name['cells_per_column_limit_filter'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['strip_value_transformer']) -_ROWFILTER.fields_by_name['strip_value_transformer'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_ROWFILTER.oneofs_by_name['filter'].fields.append( - _ROWFILTER.fields_by_name['apply_label_transformer']) -_ROWFILTER.fields_by_name['apply_label_transformer'].containing_oneof = _ROWFILTER.oneofs_by_name['filter'] -_MUTATION_SETCELL.containing_type = _MUTATION -_MUTATION_DELETEFROMCOLUMN.fields_by_name['time_range'].message_type = _TIMESTAMPRANGE -_MUTATION_DELETEFROMCOLUMN.containing_type = _MUTATION -_MUTATION_DELETEFROMFAMILY.containing_type = _MUTATION -_MUTATION_DELETEFROMROW.containing_type = _MUTATION -_MUTATION.fields_by_name['set_cell'].message_type = _MUTATION_SETCELL -_MUTATION.fields_by_name['delete_from_column'].message_type = _MUTATION_DELETEFROMCOLUMN -_MUTATION.fields_by_name['delete_from_family'].message_type = _MUTATION_DELETEFROMFAMILY -_MUTATION.fields_by_name['delete_from_row'].message_type = _MUTATION_DELETEFROMROW -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['set_cell']) -_MUTATION.fields_by_name['set_cell'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['delete_from_column']) -_MUTATION.fields_by_name['delete_from_column'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['delete_from_family']) -_MUTATION.fields_by_name['delete_from_family'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_MUTATION.oneofs_by_name['mutation'].fields.append( - _MUTATION.fields_by_name['delete_from_row']) -_MUTATION.fields_by_name['delete_from_row'].containing_oneof = _MUTATION.oneofs_by_name['mutation'] -_READMODIFYWRITERULE.oneofs_by_name['rule'].fields.append( - _READMODIFYWRITERULE.fields_by_name['append_value']) -_READMODIFYWRITERULE.fields_by_name['append_value'].containing_oneof = _READMODIFYWRITERULE.oneofs_by_name['rule'] -_READMODIFYWRITERULE.oneofs_by_name['rule'].fields.append( - _READMODIFYWRITERULE.fields_by_name['increment_amount']) -_READMODIFYWRITERULE.fields_by_name['increment_amount'].containing_oneof = _READMODIFYWRITERULE.oneofs_by_name['rule'] -DESCRIPTOR.message_types_by_name['Row'] = _ROW -DESCRIPTOR.message_types_by_name['Family'] = _FAMILY -DESCRIPTOR.message_types_by_name['Column'] = _COLUMN -DESCRIPTOR.message_types_by_name['Cell'] = _CELL -DESCRIPTOR.message_types_by_name['RowRange'] = _ROWRANGE -DESCRIPTOR.message_types_by_name['ColumnRange'] = _COLUMNRANGE -DESCRIPTOR.message_types_by_name['TimestampRange'] = _TIMESTAMPRANGE -DESCRIPTOR.message_types_by_name['ValueRange'] = _VALUERANGE -DESCRIPTOR.message_types_by_name['RowFilter'] = _ROWFILTER -DESCRIPTOR.message_types_by_name['Mutation'] = _MUTATION -DESCRIPTOR.message_types_by_name['ReadModifyWriteRule'] = _READMODIFYWRITERULE - -Row = _reflection.GeneratedProtocolMessageType('Row', (_message.Message,), dict( - DESCRIPTOR = _ROW, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Row) - )) -_sym_db.RegisterMessage(Row) - -Family = _reflection.GeneratedProtocolMessageType('Family', (_message.Message,), dict( - DESCRIPTOR = _FAMILY, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Family) - )) -_sym_db.RegisterMessage(Family) - -Column = _reflection.GeneratedProtocolMessageType('Column', (_message.Message,), dict( - DESCRIPTOR = _COLUMN, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Column) - )) -_sym_db.RegisterMessage(Column) - -Cell = _reflection.GeneratedProtocolMessageType('Cell', (_message.Message,), dict( - DESCRIPTOR = _CELL, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Cell) - )) -_sym_db.RegisterMessage(Cell) - -RowRange = _reflection.GeneratedProtocolMessageType('RowRange', (_message.Message,), dict( - DESCRIPTOR = _ROWRANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowRange) - )) -_sym_db.RegisterMessage(RowRange) - -ColumnRange = _reflection.GeneratedProtocolMessageType('ColumnRange', (_message.Message,), dict( - DESCRIPTOR = _COLUMNRANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ColumnRange) - )) -_sym_db.RegisterMessage(ColumnRange) - -TimestampRange = _reflection.GeneratedProtocolMessageType('TimestampRange', (_message.Message,), dict( - DESCRIPTOR = _TIMESTAMPRANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.TimestampRange) - )) -_sym_db.RegisterMessage(TimestampRange) - -ValueRange = _reflection.GeneratedProtocolMessageType('ValueRange', (_message.Message,), dict( - DESCRIPTOR = _VALUERANGE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ValueRange) - )) -_sym_db.RegisterMessage(ValueRange) - -RowFilter = _reflection.GeneratedProtocolMessageType('RowFilter', (_message.Message,), dict( - - Chain = _reflection.GeneratedProtocolMessageType('Chain', (_message.Message,), dict( - DESCRIPTOR = _ROWFILTER_CHAIN, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter.Chain) - )) - , - - Interleave = _reflection.GeneratedProtocolMessageType('Interleave', (_message.Message,), dict( - DESCRIPTOR = _ROWFILTER_INTERLEAVE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter.Interleave) - )) - , - - Condition = _reflection.GeneratedProtocolMessageType('Condition', (_message.Message,), dict( - DESCRIPTOR = _ROWFILTER_CONDITION, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter.Condition) - )) - , - DESCRIPTOR = _ROWFILTER, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.RowFilter) - )) -_sym_db.RegisterMessage(RowFilter) -_sym_db.RegisterMessage(RowFilter.Chain) -_sym_db.RegisterMessage(RowFilter.Interleave) -_sym_db.RegisterMessage(RowFilter.Condition) - -Mutation = _reflection.GeneratedProtocolMessageType('Mutation', (_message.Message,), dict( - - SetCell = _reflection.GeneratedProtocolMessageType('SetCell', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_SETCELL, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.SetCell) - )) - , - - DeleteFromColumn = _reflection.GeneratedProtocolMessageType('DeleteFromColumn', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_DELETEFROMCOLUMN, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.DeleteFromColumn) - )) - , - - DeleteFromFamily = _reflection.GeneratedProtocolMessageType('DeleteFromFamily', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_DELETEFROMFAMILY, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.DeleteFromFamily) - )) - , - - DeleteFromRow = _reflection.GeneratedProtocolMessageType('DeleteFromRow', (_message.Message,), dict( - DESCRIPTOR = _MUTATION_DELETEFROMROW, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation.DeleteFromRow) - )) - , - DESCRIPTOR = _MUTATION, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.Mutation) - )) -_sym_db.RegisterMessage(Mutation) -_sym_db.RegisterMessage(Mutation.SetCell) -_sym_db.RegisterMessage(Mutation.DeleteFromColumn) -_sym_db.RegisterMessage(Mutation.DeleteFromFamily) -_sym_db.RegisterMessage(Mutation.DeleteFromRow) - -ReadModifyWriteRule = _reflection.GeneratedProtocolMessageType('ReadModifyWriteRule', (_message.Message,), dict( - DESCRIPTOR = _READMODIFYWRITERULE, - __module__ = 'google.bigtable.v1.bigtable_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadModifyWriteRule) - )) -_sym_db.RegisterMessage(ReadModifyWriteRule) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.bigtable.v1B\021BigtableDataProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py deleted file mode 100644 index c07deac..0000000 --- a/gcloud_bigtable/_generated/bigtable_service_messages_pb2.py +++ /dev/null @@ -1,537 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/v1/bigtable_service_messages.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import bigtable_data_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__data__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/v1/bigtable_service_messages.proto', - package='google.bigtable.v1', - syntax='proto3', - serialized_pb=_b('\n2google/bigtable/v1/bigtable_service_messages.proto\x12\x12google.bigtable.v1\x1a&google/bigtable/v1/bigtable_data.proto\"\xdc\x01\n\x0fReadRowsRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x11\n\x07row_key\x18\x02 \x01(\x0cH\x00\x12\x31\n\trow_range\x18\x03 \x01(\x0b\x32\x1c.google.bigtable.v1.RowRangeH\x00\x12-\n\x06\x66ilter\x18\x05 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x1e\n\x16\x61llow_row_interleaving\x18\x06 \x01(\x08\x12\x16\n\x0enum_rows_limit\x18\x07 \x01(\x03\x42\x08\n\x06target\"\xd0\x01\n\x10ReadRowsResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12:\n\x06\x63hunks\x18\x02 \x03(\x0b\x32*.google.bigtable.v1.ReadRowsResponse.Chunk\x1ao\n\x05\x43hunk\x12\x32\n\x0crow_contents\x18\x01 \x01(\x0b\x32\x1a.google.bigtable.v1.FamilyH\x00\x12\x13\n\treset_row\x18\x02 \x01(\x08H\x00\x12\x14\n\ncommit_row\x18\x03 \x01(\x08H\x00\x42\x07\n\x05\x63hunk\"*\n\x14SampleRowKeysRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\">\n\x15SampleRowKeysResponse\x12\x0f\n\x07row_key\x18\x01 \x01(\x0c\x12\x14\n\x0coffset_bytes\x18\x02 \x01(\x03\"h\n\x10MutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12/\n\tmutations\x18\x03 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"\xe5\x01\n\x18\x43heckAndMutateRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x37\n\x10predicate_filter\x18\x06 \x01(\x0b\x32\x1d.google.bigtable.v1.RowFilter\x12\x34\n\x0etrue_mutations\x18\x04 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\x12\x35\n\x0f\x66\x61lse_mutations\x18\x05 \x03(\x0b\x32\x1c.google.bigtable.v1.Mutation\"6\n\x19\x43heckAndMutateRowResponse\x12\x19\n\x11predicate_matched\x18\x01 \x01(\x08\"x\n\x19ReadModifyWriteRowRequest\x12\x12\n\ntable_name\x18\x01 \x01(\t\x12\x0f\n\x07row_key\x18\x02 \x01(\x0c\x12\x36\n\x05rules\x18\x03 \x03(\x0b\x32\'.google.bigtable.v1.ReadModifyWriteRuleB8\n\x16\x63om.google.bigtable.v1B\x1c\x42igtableServiceMessagesProtoP\x01\x62\x06proto3') - , - dependencies=[google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_READROWSREQUEST = _descriptor.Descriptor( - name='ReadRowsRequest', - full_name='google.bigtable.v1.ReadRowsRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.ReadRowsRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.ReadRowsRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_range', full_name='google.bigtable.v1.ReadRowsRequest.row_range', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='filter', full_name='google.bigtable.v1.ReadRowsRequest.filter', index=3, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='allow_row_interleaving', full_name='google.bigtable.v1.ReadRowsRequest.allow_row_interleaving', index=4, - number=6, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='num_rows_limit', full_name='google.bigtable.v1.ReadRowsRequest.num_rows_limit', index=5, - number=7, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='target', full_name='google.bigtable.v1.ReadRowsRequest.target', - index=0, containing_type=None, fields=[]), - ], - serialized_start=115, - serialized_end=335, -) - - -_READROWSRESPONSE_CHUNK = _descriptor.Descriptor( - name='Chunk', - full_name='google.bigtable.v1.ReadRowsResponse.Chunk', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_contents', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.row_contents', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='reset_row', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.reset_row', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='commit_row', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.commit_row', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='chunk', full_name='google.bigtable.v1.ReadRowsResponse.Chunk.chunk', - index=0, containing_type=None, fields=[]), - ], - serialized_start=435, - serialized_end=546, -) - -_READROWSRESPONSE = _descriptor.Descriptor( - name='ReadRowsResponse', - full_name='google.bigtable.v1.ReadRowsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.ReadRowsResponse.row_key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='chunks', full_name='google.bigtable.v1.ReadRowsResponse.chunks', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_READROWSRESPONSE_CHUNK, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=338, - serialized_end=546, -) - - -_SAMPLEROWKEYSREQUEST = _descriptor.Descriptor( - name='SampleRowKeysRequest', - full_name='google.bigtable.v1.SampleRowKeysRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.SampleRowKeysRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=548, - serialized_end=590, -) - - -_SAMPLEROWKEYSRESPONSE = _descriptor.Descriptor( - name='SampleRowKeysResponse', - full_name='google.bigtable.v1.SampleRowKeysResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.SampleRowKeysResponse.row_key', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='offset_bytes', full_name='google.bigtable.v1.SampleRowKeysResponse.offset_bytes', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=592, - serialized_end=654, -) - - -_MUTATEROWREQUEST = _descriptor.Descriptor( - name='MutateRowRequest', - full_name='google.bigtable.v1.MutateRowRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.MutateRowRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.MutateRowRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='mutations', full_name='google.bigtable.v1.MutateRowRequest.mutations', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=656, - serialized_end=760, -) - - -_CHECKANDMUTATEROWREQUEST = _descriptor.Descriptor( - name='CheckAndMutateRowRequest', - full_name='google.bigtable.v1.CheckAndMutateRowRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.CheckAndMutateRowRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.CheckAndMutateRowRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='predicate_filter', full_name='google.bigtable.v1.CheckAndMutateRowRequest.predicate_filter', index=2, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='true_mutations', full_name='google.bigtable.v1.CheckAndMutateRowRequest.true_mutations', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='false_mutations', full_name='google.bigtable.v1.CheckAndMutateRowRequest.false_mutations', index=4, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=763, - serialized_end=992, -) - - -_CHECKANDMUTATEROWRESPONSE = _descriptor.Descriptor( - name='CheckAndMutateRowResponse', - full_name='google.bigtable.v1.CheckAndMutateRowResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='predicate_matched', full_name='google.bigtable.v1.CheckAndMutateRowResponse.predicate_matched', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=994, - serialized_end=1048, -) - - -_READMODIFYWRITEROWREQUEST = _descriptor.Descriptor( - name='ReadModifyWriteRowRequest', - full_name='google.bigtable.v1.ReadModifyWriteRowRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='table_name', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.table_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='row_key', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.row_key', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='rules', full_name='google.bigtable.v1.ReadModifyWriteRowRequest.rules', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1050, - serialized_end=1170, -) - -_READROWSREQUEST.fields_by_name['row_range'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWRANGE -_READROWSREQUEST.fields_by_name['filter'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWFILTER -_READROWSREQUEST.oneofs_by_name['target'].fields.append( - _READROWSREQUEST.fields_by_name['row_key']) -_READROWSREQUEST.fields_by_name['row_key'].containing_oneof = _READROWSREQUEST.oneofs_by_name['target'] -_READROWSREQUEST.oneofs_by_name['target'].fields.append( - _READROWSREQUEST.fields_by_name['row_range']) -_READROWSREQUEST.fields_by_name['row_range'].containing_oneof = _READROWSREQUEST.oneofs_by_name['target'] -_READROWSRESPONSE_CHUNK.fields_by_name['row_contents'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._FAMILY -_READROWSRESPONSE_CHUNK.containing_type = _READROWSRESPONSE -_READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'].fields.append( - _READROWSRESPONSE_CHUNK.fields_by_name['row_contents']) -_READROWSRESPONSE_CHUNK.fields_by_name['row_contents'].containing_oneof = _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'] -_READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'].fields.append( - _READROWSRESPONSE_CHUNK.fields_by_name['reset_row']) -_READROWSRESPONSE_CHUNK.fields_by_name['reset_row'].containing_oneof = _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'] -_READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'].fields.append( - _READROWSRESPONSE_CHUNK.fields_by_name['commit_row']) -_READROWSRESPONSE_CHUNK.fields_by_name['commit_row'].containing_oneof = _READROWSRESPONSE_CHUNK.oneofs_by_name['chunk'] -_READROWSRESPONSE.fields_by_name['chunks'].message_type = _READROWSRESPONSE_CHUNK -_MUTATEROWREQUEST.fields_by_name['mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION -_CHECKANDMUTATEROWREQUEST.fields_by_name['predicate_filter'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._ROWFILTER -_CHECKANDMUTATEROWREQUEST.fields_by_name['true_mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION -_CHECKANDMUTATEROWREQUEST.fields_by_name['false_mutations'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._MUTATION -_READMODIFYWRITEROWREQUEST.fields_by_name['rules'].message_type = google_dot_bigtable_dot_v1_dot_bigtable__data__pb2._READMODIFYWRITERULE -DESCRIPTOR.message_types_by_name['ReadRowsRequest'] = _READROWSREQUEST -DESCRIPTOR.message_types_by_name['ReadRowsResponse'] = _READROWSRESPONSE -DESCRIPTOR.message_types_by_name['SampleRowKeysRequest'] = _SAMPLEROWKEYSREQUEST -DESCRIPTOR.message_types_by_name['SampleRowKeysResponse'] = _SAMPLEROWKEYSRESPONSE -DESCRIPTOR.message_types_by_name['MutateRowRequest'] = _MUTATEROWREQUEST -DESCRIPTOR.message_types_by_name['CheckAndMutateRowRequest'] = _CHECKANDMUTATEROWREQUEST -DESCRIPTOR.message_types_by_name['CheckAndMutateRowResponse'] = _CHECKANDMUTATEROWRESPONSE -DESCRIPTOR.message_types_by_name['ReadModifyWriteRowRequest'] = _READMODIFYWRITEROWREQUEST - -ReadRowsRequest = _reflection.GeneratedProtocolMessageType('ReadRowsRequest', (_message.Message,), dict( - DESCRIPTOR = _READROWSREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadRowsRequest) - )) -_sym_db.RegisterMessage(ReadRowsRequest) - -ReadRowsResponse = _reflection.GeneratedProtocolMessageType('ReadRowsResponse', (_message.Message,), dict( - - Chunk = _reflection.GeneratedProtocolMessageType('Chunk', (_message.Message,), dict( - DESCRIPTOR = _READROWSRESPONSE_CHUNK, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadRowsResponse.Chunk) - )) - , - DESCRIPTOR = _READROWSRESPONSE, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadRowsResponse) - )) -_sym_db.RegisterMessage(ReadRowsResponse) -_sym_db.RegisterMessage(ReadRowsResponse.Chunk) - -SampleRowKeysRequest = _reflection.GeneratedProtocolMessageType('SampleRowKeysRequest', (_message.Message,), dict( - DESCRIPTOR = _SAMPLEROWKEYSREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.SampleRowKeysRequest) - )) -_sym_db.RegisterMessage(SampleRowKeysRequest) - -SampleRowKeysResponse = _reflection.GeneratedProtocolMessageType('SampleRowKeysResponse', (_message.Message,), dict( - DESCRIPTOR = _SAMPLEROWKEYSRESPONSE, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.SampleRowKeysResponse) - )) -_sym_db.RegisterMessage(SampleRowKeysResponse) - -MutateRowRequest = _reflection.GeneratedProtocolMessageType('MutateRowRequest', (_message.Message,), dict( - DESCRIPTOR = _MUTATEROWREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.MutateRowRequest) - )) -_sym_db.RegisterMessage(MutateRowRequest) - -CheckAndMutateRowRequest = _reflection.GeneratedProtocolMessageType('CheckAndMutateRowRequest', (_message.Message,), dict( - DESCRIPTOR = _CHECKANDMUTATEROWREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.CheckAndMutateRowRequest) - )) -_sym_db.RegisterMessage(CheckAndMutateRowRequest) - -CheckAndMutateRowResponse = _reflection.GeneratedProtocolMessageType('CheckAndMutateRowResponse', (_message.Message,), dict( - DESCRIPTOR = _CHECKANDMUTATEROWRESPONSE, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.CheckAndMutateRowResponse) - )) -_sym_db.RegisterMessage(CheckAndMutateRowResponse) - -ReadModifyWriteRowRequest = _reflection.GeneratedProtocolMessageType('ReadModifyWriteRowRequest', (_message.Message,), dict( - DESCRIPTOR = _READMODIFYWRITEROWREQUEST, - __module__ = 'google.bigtable.v1.bigtable_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.v1.ReadModifyWriteRowRequest) - )) -_sym_db.RegisterMessage(ReadModifyWriteRowRequest) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.bigtable.v1B\034BigtableServiceMessagesProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_service_pb2.py b/gcloud_bigtable/_generated/bigtable_service_pb2.py deleted file mode 100644 index 4e7e7e4..0000000 --- a/gcloud_bigtable/_generated/bigtable_service_pb2.py +++ /dev/null @@ -1,163 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/v1/bigtable_service.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import bigtable_data_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__data__pb2 -from gcloud_bigtable._generated import bigtable_service_messages_pb2 as google_dot_bigtable_dot_v1_dot_bigtable__service__messages__pb2 -from gcloud_bigtable._generated import empty_pb2 as google_dot_protobuf_dot_empty__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/v1/bigtable_service.proto', - package='google.bigtable.v1', - syntax='proto3', - serialized_pb=_b('\n)google/bigtable/v1/bigtable_service.proto\x12\x12google.bigtable.v1\x1a\x1cgoogle/api/annotations.proto\x1a&google/bigtable/v1/bigtable_data.proto\x1a\x32google/bigtable/v1/bigtable_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\xb0\x07\n\x0f\x42igtableService\x12\xa5\x01\n\x08ReadRows\x12#.google.bigtable.v1.ReadRowsRequest\x1a$.google.bigtable.v1.ReadRowsResponse\"L\x82\xd3\xe4\x93\x02\x46\"A/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:read:\x01*0\x01\x12\xb7\x01\n\rSampleRowKeys\x12(.google.bigtable.v1.SampleRowKeysRequest\x1a).google.bigtable.v1.SampleRowKeysResponse\"O\x82\xd3\xe4\x93\x02I\x12G/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows:sampleKeys0\x01\x12\xa3\x01\n\tMutateRow\x12$.google.bigtable.v1.MutateRowRequest\x1a\x16.google.protobuf.Empty\"X\x82\xd3\xe4\x93\x02R\"M/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:mutate:\x01*\x12\xd2\x01\n\x11\x43heckAndMutateRow\x12,.google.bigtable.v1.CheckAndMutateRowRequest\x1a-.google.bigtable.v1.CheckAndMutateRowResponse\"`\x82\xd3\xe4\x93\x02Z\"U/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:checkAndMutate:\x01*\x12\xbf\x01\n\x12ReadModifyWriteRow\x12-.google.bigtable.v1.ReadModifyWriteRowRequest\x1a\x17.google.bigtable.v1.Row\"a\x82\xd3\xe4\x93\x02[\"V/v1/{table_name=projects/*/zones/*/clusters/*/tables/*}/rows/{row_key}:readModifyWrite:\x01*B4\n\x16\x63om.google.bigtable.v1B\x15\x42igtableServicesProtoP\x01\x88\x01\x01\x62\x06proto3') - , - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__data__pb2.DESCRIPTOR,google_dot_bigtable_dot_v1_dot_bigtable__service__messages__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.bigtable.v1B\025BigtableServicesProtoP\001\210\001\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -class EarlyAdopterBigtableServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ReadRows(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def SampleRowKeys(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def MutateRow(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CheckAndMutateRow(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ReadModifyWriteRow(self, request, context): - raise NotImplementedError() -class EarlyAdopterBigtableServiceServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterBigtableServiceStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def ReadRows(self, request): - raise NotImplementedError() - ReadRows.async = None - @abc.abstractmethod - def SampleRowKeys(self, request): - raise NotImplementedError() - SampleRowKeys.async = None - @abc.abstractmethod - def MutateRow(self, request): - raise NotImplementedError() - MutateRow.async = None - @abc.abstractmethod - def CheckAndMutateRow(self, request): - raise NotImplementedError() - CheckAndMutateRow.async = None - @abc.abstractmethod - def ReadModifyWriteRow(self, request): - raise NotImplementedError() - ReadModifyWriteRow.async = None -def early_adopter_create_BigtableService_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_data_pb2 - method_service_descriptions = { - "CheckAndMutateRow": utilities.unary_unary_service_description( - servicer.CheckAndMutateRow, - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.FromString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.SerializeToString, - ), - "MutateRow": utilities.unary_unary_service_description( - servicer.MutateRow, - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "ReadModifyWriteRow": utilities.unary_unary_service_description( - servicer.ReadModifyWriteRow, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.FromString, - gcloud_bigtable._generated.bigtable_data_pb2.Row.SerializeToString, - ), - "ReadRows": utilities.unary_stream_service_description( - servicer.ReadRows, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.FromString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.SerializeToString, - ), - "SampleRowKeys": utilities.unary_stream_service_description( - servicer.SampleRowKeys, - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.FromString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.SerializeToString, - ), - } - return implementations.server("google.bigtable.v1.BigtableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_BigtableService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_data_pb2 - method_invocation_descriptions = { - "CheckAndMutateRow": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.CheckAndMutateRowResponse.FromString, - ), - "MutateRow": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.MutateRowRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "ReadModifyWriteRow": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadModifyWriteRowRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_data_pb2.Row.FromString, - ), - "ReadRows": utilities.unary_stream_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.ReadRowsResponse.FromString, - ), - "SampleRowKeys": utilities.unary_stream_invocation_description( - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_service_messages_pb2.SampleRowKeysResponse.FromString, - ), - } - return implementations.stub("google.bigtable.v1.BigtableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py b/gcloud_bigtable/_generated/bigtable_table_data_pb2.py deleted file mode 100644 index 22cc067..0000000 --- a/gcloud_bigtable/_generated/bigtable_table_data_pb2.py +++ /dev/null @@ -1,382 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/table/v1/bigtable_table_data.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import operations_pb2 as google_dot_longrunning_dot_operations__pb2 -from gcloud_bigtable._generated import duration_pb2 as google_dot_protobuf_dot_duration__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/table/v1/bigtable_table_data.proto', - package='google.bigtable.admin.table.v1', - syntax='proto3', - serialized_pb=_b('\n8google/bigtable/admin/table/v1/bigtable_table_data.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a#google/longrunning/operations.proto\x1a\x1egoogle/protobuf/duration.proto\"\xfd\x02\n\x05Table\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x38\n\x11\x63urrent_operation\x18\x02 \x01(\x0b\x32\x1d.google.longrunning.Operation\x12R\n\x0f\x63olumn_families\x18\x03 \x03(\x0b\x32\x39.google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry\x12O\n\x0bgranularity\x18\x04 \x01(\x0e\x32:.google.bigtable.admin.table.v1.Table.TimestampGranularity\x1a\x63\n\x13\x43olumnFamiliesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12;\n\x05value\x18\x02 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily:\x02\x38\x01\"\"\n\x14TimestampGranularity\x12\n\n\x06MILLIS\x10\x00\"l\n\x0c\x43olumnFamily\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\rgc_expression\x18\x02 \x01(\t\x12\x37\n\x07gc_rule\x18\x03 \x01(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\"\xed\x02\n\x06GcRule\x12\x1a\n\x10max_num_versions\x18\x01 \x01(\x05H\x00\x12,\n\x07max_age\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationH\x00\x12K\n\x0cintersection\x18\x03 \x01(\x0b\x32\x33.google.bigtable.admin.table.v1.GcRule.IntersectionH\x00\x12=\n\x05union\x18\x04 \x01(\x0b\x32,.google.bigtable.admin.table.v1.GcRule.UnionH\x00\x1a\x45\n\x0cIntersection\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRule\x1a>\n\x05Union\x12\x35\n\x05rules\x18\x01 \x03(\x0b\x32&.google.bigtable.admin.table.v1.GcRuleB\x06\n\x04ruleB>\n\"com.google.bigtable.admin.table.v1B\x16\x42igtableTableDataProtoP\x01\x62\x06proto3') - , - dependencies=[google_dot_longrunning_dot_operations__pb2.DESCRIPTOR,google_dot_protobuf_dot_duration__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - -_TABLE_TIMESTAMPGRANULARITY = _descriptor.EnumDescriptor( - name='TimestampGranularity', - full_name='google.bigtable.admin.table.v1.Table.TimestampGranularity', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='MILLIS', index=0, number=0, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=509, - serialized_end=543, -) -_sym_db.RegisterEnumDescriptor(_TABLE_TIMESTAMPGRANULARITY) - - -_TABLE_COLUMNFAMILIESENTRY = _descriptor.Descriptor( - name='ColumnFamiliesEntry', - full_name='google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=408, - serialized_end=507, -) - -_TABLE = _descriptor.Descriptor( - name='Table', - full_name='google.bigtable.admin.table.v1.Table', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.Table.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='current_operation', full_name='google.bigtable.admin.table.v1.Table.current_operation', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_families', full_name='google.bigtable.admin.table.v1.Table.column_families', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='granularity', full_name='google.bigtable.admin.table.v1.Table.granularity', index=3, - number=4, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_TABLE_COLUMNFAMILIESENTRY, ], - enum_types=[ - _TABLE_TIMESTAMPGRANULARITY, - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=162, - serialized_end=543, -) - - -_COLUMNFAMILY = _descriptor.Descriptor( - name='ColumnFamily', - full_name='google.bigtable.admin.table.v1.ColumnFamily', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.ColumnFamily.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='gc_expression', full_name='google.bigtable.admin.table.v1.ColumnFamily.gc_expression', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='gc_rule', full_name='google.bigtable.admin.table.v1.ColumnFamily.gc_rule', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=545, - serialized_end=653, -) - - -_GCRULE_INTERSECTION = _descriptor.Descriptor( - name='Intersection', - full_name='google.bigtable.admin.table.v1.GcRule.Intersection', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='rules', full_name='google.bigtable.admin.table.v1.GcRule.Intersection.rules', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=880, - serialized_end=949, -) - -_GCRULE_UNION = _descriptor.Descriptor( - name='Union', - full_name='google.bigtable.admin.table.v1.GcRule.Union', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='rules', full_name='google.bigtable.admin.table.v1.GcRule.Union.rules', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=951, - serialized_end=1013, -) - -_GCRULE = _descriptor.Descriptor( - name='GcRule', - full_name='google.bigtable.admin.table.v1.GcRule', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='max_num_versions', full_name='google.bigtable.admin.table.v1.GcRule.max_num_versions', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='max_age', full_name='google.bigtable.admin.table.v1.GcRule.max_age', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='intersection', full_name='google.bigtable.admin.table.v1.GcRule.intersection', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='union', full_name='google.bigtable.admin.table.v1.GcRule.union', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_GCRULE_INTERSECTION, _GCRULE_UNION, ], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='rule', full_name='google.bigtable.admin.table.v1.GcRule.rule', - index=0, containing_type=None, fields=[]), - ], - serialized_start=656, - serialized_end=1021, -) - -_TABLE_COLUMNFAMILIESENTRY.fields_by_name['value'].message_type = _COLUMNFAMILY -_TABLE_COLUMNFAMILIESENTRY.containing_type = _TABLE -_TABLE.fields_by_name['current_operation'].message_type = google_dot_longrunning_dot_operations__pb2._OPERATION -_TABLE.fields_by_name['column_families'].message_type = _TABLE_COLUMNFAMILIESENTRY -_TABLE.fields_by_name['granularity'].enum_type = _TABLE_TIMESTAMPGRANULARITY -_TABLE_TIMESTAMPGRANULARITY.containing_type = _TABLE -_COLUMNFAMILY.fields_by_name['gc_rule'].message_type = _GCRULE -_GCRULE_INTERSECTION.fields_by_name['rules'].message_type = _GCRULE -_GCRULE_INTERSECTION.containing_type = _GCRULE -_GCRULE_UNION.fields_by_name['rules'].message_type = _GCRULE -_GCRULE_UNION.containing_type = _GCRULE -_GCRULE.fields_by_name['max_age'].message_type = google_dot_protobuf_dot_duration__pb2._DURATION -_GCRULE.fields_by_name['intersection'].message_type = _GCRULE_INTERSECTION -_GCRULE.fields_by_name['union'].message_type = _GCRULE_UNION -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['max_num_versions']) -_GCRULE.fields_by_name['max_num_versions'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['max_age']) -_GCRULE.fields_by_name['max_age'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['intersection']) -_GCRULE.fields_by_name['intersection'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -_GCRULE.oneofs_by_name['rule'].fields.append( - _GCRULE.fields_by_name['union']) -_GCRULE.fields_by_name['union'].containing_oneof = _GCRULE.oneofs_by_name['rule'] -DESCRIPTOR.message_types_by_name['Table'] = _TABLE -DESCRIPTOR.message_types_by_name['ColumnFamily'] = _COLUMNFAMILY -DESCRIPTOR.message_types_by_name['GcRule'] = _GCRULE - -Table = _reflection.GeneratedProtocolMessageType('Table', (_message.Message,), dict( - - ColumnFamiliesEntry = _reflection.GeneratedProtocolMessageType('ColumnFamiliesEntry', (_message.Message,), dict( - DESCRIPTOR = _TABLE_COLUMNFAMILIESENTRY, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.Table.ColumnFamiliesEntry) - )) - , - DESCRIPTOR = _TABLE, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.Table) - )) -_sym_db.RegisterMessage(Table) -_sym_db.RegisterMessage(Table.ColumnFamiliesEntry) - -ColumnFamily = _reflection.GeneratedProtocolMessageType('ColumnFamily', (_message.Message,), dict( - DESCRIPTOR = _COLUMNFAMILY, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.ColumnFamily) - )) -_sym_db.RegisterMessage(ColumnFamily) - -GcRule = _reflection.GeneratedProtocolMessageType('GcRule', (_message.Message,), dict( - - Intersection = _reflection.GeneratedProtocolMessageType('Intersection', (_message.Message,), dict( - DESCRIPTOR = _GCRULE_INTERSECTION, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GcRule.Intersection) - )) - , - - Union = _reflection.GeneratedProtocolMessageType('Union', (_message.Message,), dict( - DESCRIPTOR = _GCRULE_UNION, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GcRule.Union) - )) - , - DESCRIPTOR = _GCRULE, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_data_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GcRule) - )) -_sym_db.RegisterMessage(GcRule) -_sym_db.RegisterMessage(GcRule.Intersection) -_sym_db.RegisterMessage(GcRule.Union) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\"com.google.bigtable.admin.table.v1B\026BigtableTableDataProtoP\001')) -_TABLE_COLUMNFAMILIESENTRY.has_options = True -_TABLE_COLUMNFAMILIESENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py deleted file mode 100644 index 30d8703..0000000 --- a/gcloud_bigtable/_generated/bigtable_table_service_messages_pb2.py +++ /dev/null @@ -1,394 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/table/v1/bigtable_table_service_messages.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import bigtable_table_data_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/table/v1/bigtable_table_service_messages.proto', - package='google.bigtable.admin.table.v1', - syntax='proto3', - serialized_pb=_b('\nDgoogle/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\"\x86\x01\n\x12\x43reateTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08table_id\x18\x02 \x01(\t\x12\x34\n\x05table\x18\x03 \x01(\x0b\x32%.google.bigtable.admin.table.v1.Table\x12\x1a\n\x12initial_split_keys\x18\x04 \x03(\t\"!\n\x11ListTablesRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"K\n\x12ListTablesResponse\x12\x35\n\x06tables\x18\x01 \x03(\x0b\x32%.google.bigtable.admin.table.v1.Table\"\x1f\n\x0fGetTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\"\n\x12\x44\x65leteTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"2\n\x12RenameTableRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06new_id\x18\x02 \x01(\t\"\x88\x01\n\x19\x43reateColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x18\n\x10\x63olumn_family_id\x18\x02 \x01(\t\x12\x43\n\rcolumn_family\x18\x03 \x01(\x0b\x32,.google.bigtable.admin.table.v1.ColumnFamily\")\n\x19\x44\x65leteColumnFamilyRequest\x12\x0c\n\x04name\x18\x01 \x01(\tBI\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\x01\x62\x06proto3') - , - dependencies=[google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_CREATETABLEREQUEST = _descriptor.Descriptor( - name='CreateTableRequest', - full_name='google.bigtable.admin.table.v1.CreateTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.CreateTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='table_id', full_name='google.bigtable.admin.table.v1.CreateTableRequest.table_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='table', full_name='google.bigtable.admin.table.v1.CreateTableRequest.table', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='initial_split_keys', full_name='google.bigtable.admin.table.v1.CreateTableRequest.initial_split_keys', index=3, - number=4, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=163, - serialized_end=297, -) - - -_LISTTABLESREQUEST = _descriptor.Descriptor( - name='ListTablesRequest', - full_name='google.bigtable.admin.table.v1.ListTablesRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.ListTablesRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=299, - serialized_end=332, -) - - -_LISTTABLESRESPONSE = _descriptor.Descriptor( - name='ListTablesResponse', - full_name='google.bigtable.admin.table.v1.ListTablesResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='tables', full_name='google.bigtable.admin.table.v1.ListTablesResponse.tables', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=334, - serialized_end=409, -) - - -_GETTABLEREQUEST = _descriptor.Descriptor( - name='GetTableRequest', - full_name='google.bigtable.admin.table.v1.GetTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.GetTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=411, - serialized_end=442, -) - - -_DELETETABLEREQUEST = _descriptor.Descriptor( - name='DeleteTableRequest', - full_name='google.bigtable.admin.table.v1.DeleteTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.DeleteTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=444, - serialized_end=478, -) - - -_RENAMETABLEREQUEST = _descriptor.Descriptor( - name='RenameTableRequest', - full_name='google.bigtable.admin.table.v1.RenameTableRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.RenameTableRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='new_id', full_name='google.bigtable.admin.table.v1.RenameTableRequest.new_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=480, - serialized_end=530, -) - - -_CREATECOLUMNFAMILYREQUEST = _descriptor.Descriptor( - name='CreateColumnFamilyRequest', - full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_family_id', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.column_family_id', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='column_family', full_name='google.bigtable.admin.table.v1.CreateColumnFamilyRequest.column_family', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=533, - serialized_end=669, -) - - -_DELETECOLUMNFAMILYREQUEST = _descriptor.Descriptor( - name='DeleteColumnFamilyRequest', - full_name='google.bigtable.admin.table.v1.DeleteColumnFamilyRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.bigtable.admin.table.v1.DeleteColumnFamilyRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=671, - serialized_end=712, -) - -_CREATETABLEREQUEST.fields_by_name['table'].message_type = google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2._TABLE -_LISTTABLESRESPONSE.fields_by_name['tables'].message_type = google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2._TABLE -_CREATECOLUMNFAMILYREQUEST.fields_by_name['column_family'].message_type = google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2._COLUMNFAMILY -DESCRIPTOR.message_types_by_name['CreateTableRequest'] = _CREATETABLEREQUEST -DESCRIPTOR.message_types_by_name['ListTablesRequest'] = _LISTTABLESREQUEST -DESCRIPTOR.message_types_by_name['ListTablesResponse'] = _LISTTABLESRESPONSE -DESCRIPTOR.message_types_by_name['GetTableRequest'] = _GETTABLEREQUEST -DESCRIPTOR.message_types_by_name['DeleteTableRequest'] = _DELETETABLEREQUEST -DESCRIPTOR.message_types_by_name['RenameTableRequest'] = _RENAMETABLEREQUEST -DESCRIPTOR.message_types_by_name['CreateColumnFamilyRequest'] = _CREATECOLUMNFAMILYREQUEST -DESCRIPTOR.message_types_by_name['DeleteColumnFamilyRequest'] = _DELETECOLUMNFAMILYREQUEST - -CreateTableRequest = _reflection.GeneratedProtocolMessageType('CreateTableRequest', (_message.Message,), dict( - DESCRIPTOR = _CREATETABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.CreateTableRequest) - )) -_sym_db.RegisterMessage(CreateTableRequest) - -ListTablesRequest = _reflection.GeneratedProtocolMessageType('ListTablesRequest', (_message.Message,), dict( - DESCRIPTOR = _LISTTABLESREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.ListTablesRequest) - )) -_sym_db.RegisterMessage(ListTablesRequest) - -ListTablesResponse = _reflection.GeneratedProtocolMessageType('ListTablesResponse', (_message.Message,), dict( - DESCRIPTOR = _LISTTABLESRESPONSE, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.ListTablesResponse) - )) -_sym_db.RegisterMessage(ListTablesResponse) - -GetTableRequest = _reflection.GeneratedProtocolMessageType('GetTableRequest', (_message.Message,), dict( - DESCRIPTOR = _GETTABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.GetTableRequest) - )) -_sym_db.RegisterMessage(GetTableRequest) - -DeleteTableRequest = _reflection.GeneratedProtocolMessageType('DeleteTableRequest', (_message.Message,), dict( - DESCRIPTOR = _DELETETABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.DeleteTableRequest) - )) -_sym_db.RegisterMessage(DeleteTableRequest) - -RenameTableRequest = _reflection.GeneratedProtocolMessageType('RenameTableRequest', (_message.Message,), dict( - DESCRIPTOR = _RENAMETABLEREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.RenameTableRequest) - )) -_sym_db.RegisterMessage(RenameTableRequest) - -CreateColumnFamilyRequest = _reflection.GeneratedProtocolMessageType('CreateColumnFamilyRequest', (_message.Message,), dict( - DESCRIPTOR = _CREATECOLUMNFAMILYREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.CreateColumnFamilyRequest) - )) -_sym_db.RegisterMessage(CreateColumnFamilyRequest) - -DeleteColumnFamilyRequest = _reflection.GeneratedProtocolMessageType('DeleteColumnFamilyRequest', (_message.Message,), dict( - DESCRIPTOR = _DELETECOLUMNFAMILYREQUEST, - __module__ = 'google.bigtable.admin.table.v1.bigtable_table_service_messages_pb2' - # @@protoc_insertion_point(class_scope:google.bigtable.admin.table.v1.DeleteColumnFamilyRequest) - )) -_sym_db.RegisterMessage(DeleteColumnFamilyRequest) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\"com.google.bigtable.admin.table.v1B!BigtableTableServiceMessagesProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py b/gcloud_bigtable/_generated/bigtable_table_service_pb2.py deleted file mode 100644 index 5dbf54e..0000000 --- a/gcloud_bigtable/_generated/bigtable_table_service_pb2.py +++ /dev/null @@ -1,223 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/bigtable/admin/table/v1/bigtable_table_service.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import bigtable_table_data_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2 -from gcloud_bigtable._generated import bigtable_table_service_messages_pb2 as google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__service__messages__pb2 -from gcloud_bigtable._generated import empty_pb2 as google_dot_protobuf_dot_empty__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/bigtable/admin/table/v1/bigtable_table_service.proto', - package='google.bigtable.admin.table.v1', - syntax='proto3', - serialized_pb=_b('\n;google/bigtable/admin/table/v1/bigtable_table_service.proto\x12\x1egoogle.bigtable.admin.table.v1\x1a\x1cgoogle/api/annotations.proto\x1a\x38google/bigtable/admin/table/v1/bigtable_table_data.proto\x1a\x44google/bigtable/admin/table/v1/bigtable_table_service_messages.proto\x1a\x1bgoogle/protobuf/empty.proto2\x89\x0b\n\x14\x42igtableTableService\x12\xa4\x01\n\x0b\x43reateTable\x12\x32.google.bigtable.admin.table.v1.CreateTableRequest\x1a%.google.bigtable.admin.table.v1.Table\":\x82\xd3\xe4\x93\x02\x34\"//v1/{name=projects/*/zones/*/clusters/*}/tables:\x01*\x12\xac\x01\n\nListTables\x12\x31.google.bigtable.admin.table.v1.ListTablesRequest\x1a\x32.google.bigtable.admin.table.v1.ListTablesResponse\"7\x82\xd3\xe4\x93\x02\x31\x12//v1/{name=projects/*/zones/*/clusters/*}/tables\x12\x9d\x01\n\x08GetTable\x12/.google.bigtable.admin.table.v1.GetTableRequest\x1a%.google.bigtable.admin.table.v1.Table\"9\x82\xd3\xe4\x93\x02\x33\x12\x31/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x94\x01\n\x0b\x44\x65leteTable\x12\x32.google.bigtable.admin.table.v1.DeleteTableRequest\x1a\x16.google.protobuf.Empty\"9\x82\xd3\xe4\x93\x02\x33*1/v1/{name=projects/*/zones/*/clusters/*/tables/*}\x12\x9e\x01\n\x0bRenameTable\x12\x32.google.bigtable.admin.table.v1.RenameTableRequest\x1a\x16.google.protobuf.Empty\"C\x82\xd3\xe4\x93\x02=\"8/v1/{name=projects/*/zones/*/clusters/*/tables/*}:rename:\x01*\x12\xca\x01\n\x12\x43reateColumnFamily\x12\x39.google.bigtable.admin.table.v1.CreateColumnFamilyRequest\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"K\x82\xd3\xe4\x93\x02\x45\"@/v1/{name=projects/*/zones/*/clusters/*/tables/*}/columnFamilies:\x01*\x12\xbf\x01\n\x12UpdateColumnFamily\x12,.google.bigtable.admin.table.v1.ColumnFamily\x1a,.google.bigtable.admin.table.v1.ColumnFamily\"M\x82\xd3\xe4\x93\x02G\x1a\x42/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}:\x01*\x12\xb3\x01\n\x12\x44\x65leteColumnFamily\x12\x39.google.bigtable.admin.table.v1.DeleteColumnFamilyRequest\x1a\x16.google.protobuf.Empty\"J\x82\xd3\xe4\x93\x02\x44*B/v1/{name=projects/*/zones/*/clusters/*/tables/*/columnFamilies/*}BB\n\"com.google.bigtable.admin.table.v1B\x1a\x42igtableTableServicesProtoP\x01\x62\x06proto3') - , - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__data__pb2.DESCRIPTOR,google_dot_bigtable_dot_admin_dot_table_dot_v1_dot_bigtable__table__service__messages__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\"com.google.bigtable.admin.table.v1B\032BigtableTableServicesProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -class EarlyAdopterBigtableTableServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def CreateTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListTables(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def GetTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def RenameTable(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CreateColumnFamily(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def UpdateColumnFamily(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteColumnFamily(self, request, context): - raise NotImplementedError() -class EarlyAdopterBigtableTableServiceServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterBigtableTableServiceStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def CreateTable(self, request): - raise NotImplementedError() - CreateTable.async = None - @abc.abstractmethod - def ListTables(self, request): - raise NotImplementedError() - ListTables.async = None - @abc.abstractmethod - def GetTable(self, request): - raise NotImplementedError() - GetTable.async = None - @abc.abstractmethod - def DeleteTable(self, request): - raise NotImplementedError() - DeleteTable.async = None - @abc.abstractmethod - def RenameTable(self, request): - raise NotImplementedError() - RenameTable.async = None - @abc.abstractmethod - def CreateColumnFamily(self, request): - raise NotImplementedError() - CreateColumnFamily.async = None - @abc.abstractmethod - def UpdateColumnFamily(self, request): - raise NotImplementedError() - UpdateColumnFamily.async = None - @abc.abstractmethod - def DeleteColumnFamily(self, request): - raise NotImplementedError() - DeleteColumnFamily.async = None -def early_adopter_create_BigtableTableService_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - method_service_descriptions = { - "CreateColumnFamily": utilities.unary_unary_service_description( - servicer.CreateColumnFamily, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - ), - "CreateTable": utilities.unary_unary_service_description( - servicer.CreateTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, - ), - "DeleteColumnFamily": utilities.unary_unary_service_description( - servicer.DeleteColumnFamily, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "DeleteTable": utilities.unary_unary_service_description( - servicer.DeleteTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "GetTable": utilities.unary_unary_service_description( - servicer.GetTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.SerializeToString, - ), - "ListTables": utilities.unary_unary_service_description( - servicer.ListTables, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.FromString, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.SerializeToString, - ), - "RenameTable": utilities.unary_unary_service_description( - servicer.RenameTable, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "UpdateColumnFamily": utilities.unary_unary_service_description( - servicer.UpdateColumnFamily, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - ), - } - return implementations.server("google.bigtable.admin.table.v1.BigtableTableService", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_BigtableTableService_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_data_pb2 - import gcloud_bigtable._generated.bigtable_table_service_messages_pb2 - import gcloud_bigtable._generated.empty_pb2 - method_invocation_descriptions = { - "CreateColumnFamily": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateColumnFamilyRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - ), - "CreateTable": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.CreateTableRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, - ), - "DeleteColumnFamily": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteColumnFamilyRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "DeleteTable": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.DeleteTableRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "GetTable": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.GetTableRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.Table.FromString, - ), - "ListTables": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesRequest.SerializeToString, - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.ListTablesResponse.FromString, - ), - "RenameTable": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_service_messages_pb2.RenameTableRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "UpdateColumnFamily": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.SerializeToString, - gcloud_bigtable._generated.bigtable_table_data_pb2.ColumnFamily.FromString, - ), - } - return implementations.stub("google.bigtable.admin.table.v1.BigtableTableService", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/duration_pb2.py b/gcloud_bigtable/_generated/duration_pb2.py deleted file mode 100644 index 7be5b3c..0000000 --- a/gcloud_bigtable/_generated/duration_pb2.py +++ /dev/null @@ -1,81 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/duration.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/duration.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=_b('\n\x1egoogle/protobuf/duration.proto\x12\x0fgoogle.protobuf\"*\n\x08\x44uration\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42)\n\x13\x63om.google.protobufB\rDurationProtoP\x01\xa0\x01\x01\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_DURATION = _descriptor.Descriptor( - name='Duration', - full_name='google.protobuf.Duration', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='seconds', full_name='google.protobuf.Duration.seconds', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='nanos', full_name='google.protobuf.Duration.nanos', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=51, - serialized_end=93, -) - -DESCRIPTOR.message_types_by_name['Duration'] = _DURATION - -Duration = _reflection.GeneratedProtocolMessageType('Duration', (_message.Message,), dict( - DESCRIPTOR = _DURATION, - __module__ = 'google.protobuf.duration_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Duration) - )) -_sym_db.RegisterMessage(Duration) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\rDurationProtoP\001\240\001\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/empty_pb2.py b/gcloud_bigtable/_generated/empty_pb2.py deleted file mode 100644 index 976dfb9..0000000 --- a/gcloud_bigtable/_generated/empty_pb2.py +++ /dev/null @@ -1,67 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/empty.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/empty.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=_b('\n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05\x45mptyB#\n\x13\x63om.google.protobufB\nEmptyProtoP\x01\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_EMPTY = _descriptor.Descriptor( - name='Empty', - full_name='google.protobuf.Empty', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=48, - serialized_end=55, -) - -DESCRIPTOR.message_types_by_name['Empty'] = _EMPTY - -Empty = _reflection.GeneratedProtocolMessageType('Empty', (_message.Message,), dict( - DESCRIPTOR = _EMPTY, - __module__ = 'google.protobuf.empty_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Empty) - )) -_sym_db.RegisterMessage(Empty) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\nEmptyProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/http_pb2.py b/gcloud_bigtable/_generated/http_pb2.py deleted file mode 100644 index f755e4c..0000000 --- a/gcloud_bigtable/_generated/http_pb2.py +++ /dev/null @@ -1,192 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/api/http.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/api/http.proto', - package='google.api', - syntax='proto3', - serialized_pb=_b('\n\x15google/api/http.proto\x12\ngoogle.api\"\xd8\x01\n\x08HttpRule\x12\r\n\x03get\x18\x02 \x01(\tH\x00\x12\r\n\x03put\x18\x03 \x01(\tH\x00\x12\x0e\n\x04post\x18\x04 \x01(\tH\x00\x12\x10\n\x06\x64\x65lete\x18\x05 \x01(\tH\x00\x12\x0f\n\x05patch\x18\x06 \x01(\tH\x00\x12/\n\x06\x63ustom\x18\x08 \x01(\x0b\x32\x1d.google.api.CustomHttpPatternH\x00\x12\x0c\n\x04\x62ody\x18\x07 \x01(\t\x12\x31\n\x13\x61\x64\x64itional_bindings\x18\x0b \x03(\x0b\x32\x14.google.api.HttpRuleB\t\n\x07pattern\"/\n\x11\x43ustomHttpPattern\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\tB\x1d\n\x0e\x63om.google.apiB\tHttpProtoP\x01\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_HTTPRULE = _descriptor.Descriptor( - name='HttpRule', - full_name='google.api.HttpRule', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='get', full_name='google.api.HttpRule.get', index=0, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='put', full_name='google.api.HttpRule.put', index=1, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='post', full_name='google.api.HttpRule.post', index=2, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='delete', full_name='google.api.HttpRule.delete', index=3, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='patch', full_name='google.api.HttpRule.patch', index=4, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='custom', full_name='google.api.HttpRule.custom', index=5, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='body', full_name='google.api.HttpRule.body', index=6, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='additional_bindings', full_name='google.api.HttpRule.additional_bindings', index=7, - number=11, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='pattern', full_name='google.api.HttpRule.pattern', - index=0, containing_type=None, fields=[]), - ], - serialized_start=38, - serialized_end=254, -) - - -_CUSTOMHTTPPATTERN = _descriptor.Descriptor( - name='CustomHttpPattern', - full_name='google.api.CustomHttpPattern', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='kind', full_name='google.api.CustomHttpPattern.kind', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='path', full_name='google.api.CustomHttpPattern.path', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=256, - serialized_end=303, -) - -_HTTPRULE.fields_by_name['custom'].message_type = _CUSTOMHTTPPATTERN -_HTTPRULE.fields_by_name['additional_bindings'].message_type = _HTTPRULE -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['get']) -_HTTPRULE.fields_by_name['get'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['put']) -_HTTPRULE.fields_by_name['put'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['post']) -_HTTPRULE.fields_by_name['post'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['delete']) -_HTTPRULE.fields_by_name['delete'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['patch']) -_HTTPRULE.fields_by_name['patch'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -_HTTPRULE.oneofs_by_name['pattern'].fields.append( - _HTTPRULE.fields_by_name['custom']) -_HTTPRULE.fields_by_name['custom'].containing_oneof = _HTTPRULE.oneofs_by_name['pattern'] -DESCRIPTOR.message_types_by_name['HttpRule'] = _HTTPRULE -DESCRIPTOR.message_types_by_name['CustomHttpPattern'] = _CUSTOMHTTPPATTERN - -HttpRule = _reflection.GeneratedProtocolMessageType('HttpRule', (_message.Message,), dict( - DESCRIPTOR = _HTTPRULE, - __module__ = 'google.api.http_pb2' - # @@protoc_insertion_point(class_scope:google.api.HttpRule) - )) -_sym_db.RegisterMessage(HttpRule) - -CustomHttpPattern = _reflection.GeneratedProtocolMessageType('CustomHttpPattern', (_message.Message,), dict( - DESCRIPTOR = _CUSTOMHTTPPATTERN, - __module__ = 'google.api.http_pb2' - # @@protoc_insertion_point(class_scope:google.api.CustomHttpPattern) - )) -_sym_db.RegisterMessage(CustomHttpPattern) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\016com.google.apiB\tHttpProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/operations_pb2.py b/gcloud_bigtable/_generated/operations_pb2.py deleted file mode 100644 index b31048c..0000000 --- a/gcloud_bigtable/_generated/operations_pb2.py +++ /dev/null @@ -1,446 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/longrunning/operations.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import annotations_pb2 as google_dot_api_dot_annotations__pb2 -from gcloud_bigtable._generated import any_pb2 as google_dot_protobuf_dot_any__pb2 -from gcloud_bigtable._generated import empty_pb2 as google_dot_protobuf_dot_empty__pb2 -from gcloud_bigtable._generated import status_pb2 as google_dot_rpc_dot_status__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/longrunning/operations.proto', - package='google.longrunning', - syntax='proto3', - serialized_pb=_b('\n#google/longrunning/operations.proto\x12\x12google.longrunning\x1a\x1cgoogle/api/annotations.proto\x1a\x19google/protobuf/any.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x17google/rpc/status.proto\"\xa8\x01\n\tOperation\x12\x0c\n\x04name\x18\x01 \x01(\t\x12&\n\x08metadata\x18\x02 \x01(\x0b\x32\x14.google.protobuf.Any\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12#\n\x05\x65rror\x18\x04 \x01(\x0b\x32\x12.google.rpc.StatusH\x00\x12(\n\x08response\x18\x05 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00\x42\x08\n\x06result\"#\n\x13GetOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\\\n\x15ListOperationsRequest\x12\x0c\n\x04name\x18\x04 \x01(\t\x12\x0e\n\x06\x66ilter\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"d\n\x16ListOperationsResponse\x12\x31\n\noperations\x18\x01 \x03(\x0b\x32\x1d.google.longrunning.Operation\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"&\n\x16\x43\x61ncelOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"&\n\x16\x44\x65leteOperationRequest\x12\x0c\n\x04name\x18\x01 \x01(\t2\x8c\x04\n\nOperations\x12x\n\x0cGetOperation\x12\'.google.longrunning.GetOperationRequest\x1a\x1d.google.longrunning.Operation\" \x82\xd3\xe4\x93\x02\x1a\x12\x18/v1/{name=operations/**}\x12\x86\x01\n\x0eListOperations\x12).google.longrunning.ListOperationsRequest\x1a*.google.longrunning.ListOperationsResponse\"\x1d\x82\xd3\xe4\x93\x02\x17\x12\x15/v1/{name=operations}\x12\x81\x01\n\x0f\x43\x61ncelOperation\x12*.google.longrunning.CancelOperationRequest\x1a\x16.google.protobuf.Empty\"*\x82\xd3\xe4\x93\x02$\"\x1f/v1/{name=operations/**}:cancel:\x01*\x12w\n\x0f\x44\x65leteOperation\x12*.google.longrunning.DeleteOperationRequest\x1a\x16.google.protobuf.Empty\" \x82\xd3\xe4\x93\x02\x1a*\x18/v1/{name=operations/**}B+\n\x16\x63om.google.longrunningB\x0fOperationsProtoP\x01\x62\x06proto3') - , - dependencies=[google_dot_api_dot_annotations__pb2.DESCRIPTOR,google_dot_protobuf_dot_any__pb2.DESCRIPTOR,google_dot_protobuf_dot_empty__pb2.DESCRIPTOR,google_dot_rpc_dot_status__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_OPERATION = _descriptor.Descriptor( - name='Operation', - full_name='google.longrunning.Operation', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.Operation.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='metadata', full_name='google.longrunning.Operation.metadata', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='done', full_name='google.longrunning.Operation.done', index=2, - number=3, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='error', full_name='google.longrunning.Operation.error', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='response', full_name='google.longrunning.Operation.response', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='result', full_name='google.longrunning.Operation.result', - index=0, containing_type=None, fields=[]), - ], - serialized_start=171, - serialized_end=339, -) - - -_GETOPERATIONREQUEST = _descriptor.Descriptor( - name='GetOperationRequest', - full_name='google.longrunning.GetOperationRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.GetOperationRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=341, - serialized_end=376, -) - - -_LISTOPERATIONSREQUEST = _descriptor.Descriptor( - name='ListOperationsRequest', - full_name='google.longrunning.ListOperationsRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.ListOperationsRequest.name', index=0, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='filter', full_name='google.longrunning.ListOperationsRequest.filter', index=1, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='page_size', full_name='google.longrunning.ListOperationsRequest.page_size', index=2, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='page_token', full_name='google.longrunning.ListOperationsRequest.page_token', index=3, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=378, - serialized_end=470, -) - - -_LISTOPERATIONSRESPONSE = _descriptor.Descriptor( - name='ListOperationsResponse', - full_name='google.longrunning.ListOperationsResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='operations', full_name='google.longrunning.ListOperationsResponse.operations', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='next_page_token', full_name='google.longrunning.ListOperationsResponse.next_page_token', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=472, - serialized_end=572, -) - - -_CANCELOPERATIONREQUEST = _descriptor.Descriptor( - name='CancelOperationRequest', - full_name='google.longrunning.CancelOperationRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.CancelOperationRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=574, - serialized_end=612, -) - - -_DELETEOPERATIONREQUEST = _descriptor.Descriptor( - name='DeleteOperationRequest', - full_name='google.longrunning.DeleteOperationRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.longrunning.DeleteOperationRequest.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=614, - serialized_end=652, -) - -_OPERATION.fields_by_name['metadata'].message_type = google_dot_protobuf_dot_any__pb2._ANY -_OPERATION.fields_by_name['error'].message_type = google_dot_rpc_dot_status__pb2._STATUS -_OPERATION.fields_by_name['response'].message_type = google_dot_protobuf_dot_any__pb2._ANY -_OPERATION.oneofs_by_name['result'].fields.append( - _OPERATION.fields_by_name['error']) -_OPERATION.fields_by_name['error'].containing_oneof = _OPERATION.oneofs_by_name['result'] -_OPERATION.oneofs_by_name['result'].fields.append( - _OPERATION.fields_by_name['response']) -_OPERATION.fields_by_name['response'].containing_oneof = _OPERATION.oneofs_by_name['result'] -_LISTOPERATIONSRESPONSE.fields_by_name['operations'].message_type = _OPERATION -DESCRIPTOR.message_types_by_name['Operation'] = _OPERATION -DESCRIPTOR.message_types_by_name['GetOperationRequest'] = _GETOPERATIONREQUEST -DESCRIPTOR.message_types_by_name['ListOperationsRequest'] = _LISTOPERATIONSREQUEST -DESCRIPTOR.message_types_by_name['ListOperationsResponse'] = _LISTOPERATIONSRESPONSE -DESCRIPTOR.message_types_by_name['CancelOperationRequest'] = _CANCELOPERATIONREQUEST -DESCRIPTOR.message_types_by_name['DeleteOperationRequest'] = _DELETEOPERATIONREQUEST - -Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Message,), dict( - DESCRIPTOR = _OPERATION, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.Operation) - )) -_sym_db.RegisterMessage(Operation) - -GetOperationRequest = _reflection.GeneratedProtocolMessageType('GetOperationRequest', (_message.Message,), dict( - DESCRIPTOR = _GETOPERATIONREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.GetOperationRequest) - )) -_sym_db.RegisterMessage(GetOperationRequest) - -ListOperationsRequest = _reflection.GeneratedProtocolMessageType('ListOperationsRequest', (_message.Message,), dict( - DESCRIPTOR = _LISTOPERATIONSREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.ListOperationsRequest) - )) -_sym_db.RegisterMessage(ListOperationsRequest) - -ListOperationsResponse = _reflection.GeneratedProtocolMessageType('ListOperationsResponse', (_message.Message,), dict( - DESCRIPTOR = _LISTOPERATIONSRESPONSE, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.ListOperationsResponse) - )) -_sym_db.RegisterMessage(ListOperationsResponse) - -CancelOperationRequest = _reflection.GeneratedProtocolMessageType('CancelOperationRequest', (_message.Message,), dict( - DESCRIPTOR = _CANCELOPERATIONREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.CancelOperationRequest) - )) -_sym_db.RegisterMessage(CancelOperationRequest) - -DeleteOperationRequest = _reflection.GeneratedProtocolMessageType('DeleteOperationRequest', (_message.Message,), dict( - DESCRIPTOR = _DELETEOPERATIONREQUEST, - __module__ = 'google.longrunning.operations_pb2' - # @@protoc_insertion_point(class_scope:google.longrunning.DeleteOperationRequest) - )) -_sym_db.RegisterMessage(DeleteOperationRequest) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\026com.google.longrunningB\017OperationsProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -class EarlyAdopterOperationsServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def ListOperations(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def CancelOperation(self, request, context): - raise NotImplementedError() - @abc.abstractmethod - def DeleteOperation(self, request, context): - raise NotImplementedError() -class EarlyAdopterOperationsServer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def start(self): - raise NotImplementedError() - @abc.abstractmethod - def stop(self): - raise NotImplementedError() -class EarlyAdopterOperationsStub(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def GetOperation(self, request): - raise NotImplementedError() - GetOperation.async = None - @abc.abstractmethod - def ListOperations(self, request): - raise NotImplementedError() - ListOperations.async = None - @abc.abstractmethod - def CancelOperation(self, request): - raise NotImplementedError() - CancelOperation.async = None - @abc.abstractmethod - def DeleteOperation(self, request): - raise NotImplementedError() - DeleteOperation.async = None -def early_adopter_create_Operations_server(servicer, port, private_key=None, certificate_chain=None): - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - method_service_descriptions = { - "CancelOperation": utilities.unary_unary_service_description( - servicer.CancelOperation, - gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "DeleteOperation": utilities.unary_unary_service_description( - servicer.DeleteOperation, - gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.FromString, - gcloud_bigtable._generated.empty_pb2.Empty.SerializeToString, - ), - "GetOperation": utilities.unary_unary_service_description( - servicer.GetOperation, - gcloud_bigtable._generated.operations_pb2.GetOperationRequest.FromString, - gcloud_bigtable._generated.operations_pb2.Operation.SerializeToString, - ), - "ListOperations": utilities.unary_unary_service_description( - servicer.ListOperations, - gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.FromString, - gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.SerializeToString, - ), - } - return implementations.server("google.longrunning.Operations", method_service_descriptions, port, private_key=private_key, certificate_chain=certificate_chain) -def early_adopter_create_Operations_stub(host, port, metadata_transformer=None, secure=False, root_certificates=None, private_key=None, certificate_chain=None, server_host_override=None): - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - import gcloud_bigtable._generated.operations_pb2 - import gcloud_bigtable._generated.empty_pb2 - method_invocation_descriptions = { - "CancelOperation": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.CancelOperationRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "DeleteOperation": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.DeleteOperationRequest.SerializeToString, - gcloud_bigtable._generated.empty_pb2.Empty.FromString, - ), - "GetOperation": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.GetOperationRequest.SerializeToString, - gcloud_bigtable._generated.operations_pb2.Operation.FromString, - ), - "ListOperations": utilities.unary_unary_invocation_description( - gcloud_bigtable._generated.operations_pb2.ListOperationsRequest.SerializeToString, - gcloud_bigtable._generated.operations_pb2.ListOperationsResponse.FromString, - ), - } - return implementations.stub("google.longrunning.Operations", method_invocation_descriptions, host, port, metadata_transformer=metadata_transformer, secure=secure, root_certificates=root_certificates, private_key=private_key, certificate_chain=certificate_chain, server_host_override=server_host_override) -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/status_pb2.py b/gcloud_bigtable/_generated/status_pb2.py deleted file mode 100644 index 643f286..0000000 --- a/gcloud_bigtable/_generated/status_pb2.py +++ /dev/null @@ -1,91 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/rpc/status.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from gcloud_bigtable._generated import any_pb2 as google_dot_protobuf_dot_any__pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/rpc/status.proto', - package='google.rpc', - syntax='proto3', - serialized_pb=_b('\n\x17google/rpc/status.proto\x12\ngoogle.rpc\x1a\x19google/protobuf/any.proto\"N\n\x06Status\x12\x0c\n\x04\x63ode\x18\x01 \x01(\x05\x12\x0f\n\x07message\x18\x02 \x01(\t\x12%\n\x07\x64\x65tails\x18\x03 \x03(\x0b\x32\x14.google.protobuf.AnyB\x1f\n\x0e\x63om.google.rpcB\x0bStatusProtoP\x01\x62\x06proto3') - , - dependencies=[google_dot_protobuf_dot_any__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_STATUS = _descriptor.Descriptor( - name='Status', - full_name='google.rpc.Status', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='code', full_name='google.rpc.Status.code', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='message', full_name='google.rpc.Status.message', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='details', full_name='google.rpc.Status.details', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=66, - serialized_end=144, -) - -_STATUS.fields_by_name['details'].message_type = google_dot_protobuf_dot_any__pb2._ANY -DESCRIPTOR.message_types_by_name['Status'] = _STATUS - -Status = _reflection.GeneratedProtocolMessageType('Status', (_message.Message,), dict( - DESCRIPTOR = _STATUS, - __module__ = 'google.rpc.status_pb2' - # @@protoc_insertion_point(class_scope:google.rpc.Status) - )) -_sym_db.RegisterMessage(Status) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\016com.google.rpcB\013StatusProtoP\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_generated/timestamp_pb2.py b/gcloud_bigtable/_generated/timestamp_pb2.py deleted file mode 100644 index 6baa9b4..0000000 --- a/gcloud_bigtable/_generated/timestamp_pb2.py +++ /dev/null @@ -1,81 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/timestamp.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/timestamp.proto', - package='google.protobuf', - syntax='proto3', - serialized_pb=_b('\n\x1fgoogle/protobuf/timestamp.proto\x12\x0fgoogle.protobuf\"+\n\tTimestamp\x12\x0f\n\x07seconds\x18\x01 \x01(\x03\x12\r\n\x05nanos\x18\x02 \x01(\x05\x42*\n\x13\x63om.google.protobufB\x0eTimestampProtoP\x01\xa0\x01\x01\x62\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_TIMESTAMP = _descriptor.Descriptor( - name='Timestamp', - full_name='google.protobuf.Timestamp', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='seconds', full_name='google.protobuf.Timestamp.seconds', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='nanos', full_name='google.protobuf.Timestamp.nanos', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=52, - serialized_end=95, -) - -DESCRIPTOR.message_types_by_name['Timestamp'] = _TIMESTAMP - -Timestamp = _reflection.GeneratedProtocolMessageType('Timestamp', (_message.Message,), dict( - DESCRIPTOR = _TIMESTAMP, - __module__ = 'google.protobuf.timestamp_pb2' - # @@protoc_insertion_point(class_scope:google.protobuf.Timestamp) - )) -_sym_db.RegisterMessage(Timestamp) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\023com.google.protobufB\016TimestampProtoP\001\240\001\001')) -import abc -from grpc.early_adopter import implementations -from grpc.framework.alpha import utilities -# @@protoc_insertion_point(module_scope) diff --git a/gcloud_bigtable/_grpc_mocks.py b/gcloud_bigtable/_grpc_mocks.py deleted file mode 100644 index b8c2fe2..0000000 --- a/gcloud_bigtable/_grpc_mocks.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Testing mocks for grpc.""" - - -class AsyncResult(object): - """Result returned from a ``MethodMock.async`` call.""" - - def __init__(self, result): - self._result = result - - def result(self): - """Result method on an asyc object.""" - return self._result - - -class MethodMock(object): - """Mock for :class:`grpc.framework.alpha._reexport._UnaryUnarySyncAsync`. - - May need to be callable and needs to (in our use) have an - ``async`` method. - """ - - def __init__(self, name, factory): - self._name = name - self._factory = factory - - def async(self, *args, **kwargs): - """Async method meant to mock a gRPC stub request.""" - self._factory.method_calls.append((self._name, args, kwargs)) - curr_result = self._factory.results[0] - self._factory.results = self._factory.results[1:] - return AsyncResult(curr_result) - - def __call__(self, *args, **kwargs): - """Sync method meant to mock a gRPC stub request.""" - self._factory.method_calls.append((self._name, args, kwargs)) - curr_result = self._factory.results[0] - self._factory.results = self._factory.results[1:] - return curr_result - - -class StubMock(object): - """Class to act as a gPRC stub.""" - - def __init__(self, *results): - self.results = results - self.method_calls = [] - - def __getattr__(self, name): - # We need not worry about attributes set in constructor - # since __getattribute__ will handle them. - return MethodMock(name, self) diff --git a/gcloud_bigtable/_helpers.py b/gcloud_bigtable/_helpers.py deleted file mode 100644 index 806cbd2..0000000 --- a/gcloud_bigtable/_helpers.py +++ /dev/null @@ -1,368 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility methods for gcloud_bigtable. - -Primarily includes helpers for dealing with low-level -protobuf objects. -""" - - -import datetime -import pytz -import six - -from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 -from gcloud_bigtable._generated import duration_pb2 - - -_TYPE_URL_BASE = 'type.googleapis.com/google.bigtable.' -_ADMIN_TYPE_URL_BASE = _TYPE_URL_BASE + 'admin.cluster.v1.' -_CLUSTER_TYPE_URL = _ADMIN_TYPE_URL_BASE + 'Cluster' -_CLUSTER_CREATE_METADATA = _ADMIN_TYPE_URL_BASE + 'CreateClusterMetadata' -_TYPE_URL_MAP = { - _CLUSTER_TYPE_URL: data_pb2.Cluster, - _CLUSTER_CREATE_METADATA: messages_pb2.CreateClusterMetadata, - _ADMIN_TYPE_URL_BASE + 'UndeleteClusterMetadata': ( - messages_pb2.UndeleteClusterMetadata), - _ADMIN_TYPE_URL_BASE + 'UpdateClusterMetadata': ( - messages_pb2.UpdateClusterMetadata), -} - -EPOCH = datetime.datetime.utcfromtimestamp(0).replace(tzinfo=pytz.utc) -# See https://gist.github.com/dhermes/bbc5b7be1932bfffae77 -# for appropriate values on other systems. -SSL_CERT_FILE = '/etc/ssl/certs/ca-certificates.crt' - - -class MetadataTransformer(object): - """Callable class to transform metadata for gRPC requests. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides authorization and - user agent. - """ - - def __init__(self, client): - self._credentials = client.credentials - self._user_agent = client.user_agent - - def __call__(self, ignored_val): - """Adds authorization header to request metadata.""" - access_token = self._credentials.get_access_token().access_token - return [ - ('Authorization', 'Bearer ' + access_token), - ('User-agent', self._user_agent), - ] - - -class AuthInfo(object): - """Local namespace for caching auth information.""" - - ROOT_CERTIFICATES = None - - -def _pb_timestamp_to_datetime(timestamp): - """Convert a Timestamp protobuf to a datetime object. - - :type timestamp: :class:`._generated.timestamp_pb.Timestamp` - :param timestamp: A Google returned timestamp protobuf. - - :rtype: :class:`datetime.datetime` - :returns: A UTC datetime object converted from a protobuf timestamp. - """ - return ( - EPOCH + - datetime.timedelta( - seconds=timestamp.seconds, - microseconds=(timestamp.nanos / 1000.0), - ) - ) - - -def _require_pb_property(message_pb, property_name, value): - """Check that a property agrees with the value on the message. - - :type message_pb: :class:`google.protobuf.message.Message` - :param message_pb: The message to check for ``property_name``. - - :type property_name: str - :param property_name: The property value to check against. - - :type value: object or :data:`NoneType ` - :param value: The value to check against the cluster. If :data:`None`, - will not be checked. - - :rtype: object - :returns: The value of ``property_name`` set on ``message_pb``. - :raises: :class:`ValueError ` if the result returned - from the ``message_pb`` does not contain the ``property_name`` - value or if the value returned disagrees with the ``value`` - passed with the request (if that value is not null). - """ - # Make sure `property_name` is set on the response. - # NOTE: HasField() doesn't work in protobuf>=3.0.0a3 - all_fields = set([field.name for field in message_pb._fields]) - if property_name not in all_fields: - raise ValueError('Message does not contain %s.' % (property_name,)) - property_val = getattr(message_pb, property_name) - if value is None: - value = property_val - elif value != property_val: - raise ValueError('Message returned %s value disagreeing ' - 'with value passed in.' % (property_name,)) - - return value - - -def _parse_pb_any_to_native(any_val, expected_type=None): - """Convert a serialized "google.protobuf.Any" value to actual type. - - :type any_val: :class:`gcloud_bigtable._generated.any_pb2.Any` - :param any_val: A serialized protobuf value container. - - :type expected_type: str - :param expected_type: (Optional) The type URL we expect ``any_val`` - to have. - - :rtype: object - :returns: The de-serialized object. - :raises: :class:`ValueError ` if the - ``expected_type`` does not match the ``type_url`` on the input. - """ - if expected_type is not None and expected_type != any_val.type_url: - raise ValueError('Expected type: %s, Received: %s' % ( - expected_type, any_val.type_url)) - container_class = _TYPE_URL_MAP[any_val.type_url] - return container_class.FromString(any_val.value) - - -def _timedelta_to_duration_pb(timedelta_val): - """Convert a Python timedelta object to a duration protobuf. - - .. note:: - - The Python timedelta has a granularity of microseconds while - the protobuf duration type has a duration of nanoseconds. - - :type timedelta_val: :class:`datetime.timedelta` - :param timedelta_val: A timedelta object. - - :rtype: :class:`duration_pb2.Duration` - :returns: A duration object equivalent to the time delta. - """ - seconds_decimal = timedelta_val.total_seconds() - # Truncate the parts other than the integer. - seconds = int(seconds_decimal) - if seconds_decimal < 0: - signed_micros = timedelta_val.microseconds - 10**6 - else: - signed_micros = timedelta_val.microseconds - # Convert nanoseconds to microseconds. - nanos = 1000 * signed_micros - return duration_pb2.Duration(seconds=seconds, nanos=nanos) - - -def _duration_pb_to_timedelta(duration_pb): - """Convert a duration protobuf to a Python timedelta object. - - .. note:: - - The Python timedelta has a granularity of microseconds while - the protobuf duration type has a duration of nanoseconds. - - :type duration_pb: :class:`duration_pb2.Duration` - :param duration_pb: A protobuf duration object. - - :rtype: :class:`datetime.timedelta` - :returns: The converted timedelta object. - """ - return datetime.timedelta( - seconds=duration_pb.seconds, - microseconds=(duration_pb.nanos / 1000.0), - ) - - -def _timestamp_to_microseconds(timestamp, granularity=1000): - """Converts a native datetime object to microseconds. - - .. note:: - - If ``timestamp`` does not have the same timezone as ``EPOCH`` - (which is UTC), then subtracting the epoch from the timestamp - will raise a :class:`TypeError `. - - :type timestamp: :class:`datetime.datetime` - :param timestamp: A timestamp to be converted to microseconds. - - :type granularity: int - :param granularity: The resolution (relative to microseconds) that the - timestamp should be truncated to. Defaults to 1000 - and no other value is likely needed since the only - value of the enum - :class:`.data_pb2.Table.TimestampGranularity` is - :data:`.data_pb2.Table.MILLIS`. - - :rtype: int - :returns: The ``timestamp`` as microseconds (with the appropriate - granularity). - """ - timestamp_seconds = (timestamp - EPOCH).total_seconds() - timestamp_micros = int(10**6 * timestamp_seconds) - # Truncate to granularity. - timestamp_micros -= (timestamp_micros % granularity) - return timestamp_micros - - -def _microseconds_to_timestamp(microseconds): - """Converts microseconds to a native datetime object. - - :type microseconds: int - :param microseconds: The ``timestamp`` as microseconds. - - :rtype: :class:`datetime.datetime` - :returns: A timestamp to be converted from microseconds. - """ - return EPOCH + datetime.timedelta(microseconds=microseconds) - - -def _set_certs(): - """Sets the cached root certificates locally.""" - with open(SSL_CERT_FILE, mode='rb') as file_obj: - AuthInfo.ROOT_CERTIFICATES = file_obj.read() - - -def set_certs(reset=False): - """Sets the cached root certificates locally. - - If not manually told to reset or if the value is already set, - does nothing. - - :type reset: bool - :param reset: Boolean indicating if the cached certs should be reset. - """ - if AuthInfo.ROOT_CERTIFICATES is None or reset: - _set_certs() - - -def get_certs(): - """Gets the cached root certificates. - - Calls set_certs() first in case the value has not been set, but - this will do nothing if the value is already set. - - :rtype: str - :returns: The root certificates set on ``AuthInfo``. - """ - set_certs(reset=False) - return AuthInfo.ROOT_CERTIFICATES - - -def make_stub(client, stub_factory, host, port): - """Makes a stub for the an API. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides authorization and - user agent. - - :type stub_factory: callable - :param stub_factory: A factory which will create a gRPC stub for - a given service. - - :type host: str - :param host: The host for the service. - - :type port: int - :param port: The port for the service. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: The stub object used to make gRPC requests to the - Data API. - """ - custom_metadata_transformer = MetadataTransformer(client) - return stub_factory(host, port, - metadata_transformer=custom_metadata_transformer, - secure=True, - root_certificates=get_certs()) - - -def _parse_family_pb(family_pb): - """Parses a Family protobuf into a dictionary. - - :type family_pb: :class:`._generated.bigtable_data_pb2.Family` - :param family_pb: A protobuf - - :rtype: tuple - :returns: A string and dictionary. The string is the name of the - column family and the dictionary has column names (within the - family) as keys and cell lists as values. Each cell is - represented with a two-tuple with the value (in bytes) and the - timestamp for the cell. For example: - - .. code:: python - - { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - } - """ - result = {} - for column in family_pb.columns: - result[column.qualifier] = cells = [] - for cell in column.cells: - val_pair = ( - cell.value, - _microseconds_to_timestamp(cell.timestamp_micros), - ) - cells.append(val_pair) - - return family_pb.name, result - - -def _to_bytes(value, encoding='ascii'): - """Converts a string value to bytes, if necessary. - - Unfortunately, ``six.b`` is insufficient for this task since in - Python2 it does not modify ``unicode`` objects. - - :type value: str / bytes or unicode - :param value: The string/bytes value to be converted. - - :type encoding: str - :param encoding: The encoding to use to convert unicode to bytes. Defaults - to "ascii", which will not allow any characters from - ordinals larger than 127. Other useful values are - "latin-1", which which will only allows byte ordinals - (up to 255) and "utf-8", which will encode any unicode - that needs to be. - - :rtype: str / bytes - :returns: The original value converted to bytes (if unicode) or as passed - in if it started out as bytes. - :raises: :class:`TypeError ` if the value - could not be converted to bytes. - """ - result = (value.encode(encoding) - if isinstance(value, six.text_type) else value) - if isinstance(result, six.binary_type): - return result - else: - raise TypeError('%r could not be converted to bytes' % (value,)) diff --git a/gcloud_bigtable/_logging.py b/gcloud_bigtable/_logging.py deleted file mode 100644 index 57f0591..0000000 --- a/gcloud_bigtable/_logging.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Logging settings for Google Cloud Bigtable API package.""" - - -import argparse -import logging -import os -import sys - - -LOGGER = logging.getLogger('gcloud_bigtable') -PARSER = argparse.ArgumentParser( - description='Internal gcloud-python-bigtable logging parser.') -# No help since internal. -PARSER.add_argument('--log', dest='log_level') -ENV_VAR_NAME = 'GCLOUD_LOGGING_LEVEL' - -# Get logging level from user input. -DEFAULT_LEVEL = logging.INFO -MAPPING = { - 'DEBUG': logging.DEBUG, - 'INFO': logging.INFO, - 'WARN': logging.WARN, - 'ERROR': logging.ERROR, - 'CRITICAL': logging.CRITICAL, -} - - -def get_log_level(argv): - """Get the logging level from command line arguments. - - :type argv: list of strings - :param argv: Command line arguments from caller. - - :rtype: int - :returns: The logging level parsed from the command line flags. - """ - parsed_args, _ = PARSER.parse_known_args(argv) - # Use upper case since that is used in MAPPING. - log_level = parsed_args.log_level or os.getenv(ENV_VAR_NAME) - return MAPPING.get(log_level, DEFAULT_LEVEL) - - -def setup_logger(logger, argv): - """Set the logging level on a logger. - - Uses ``get_log_level`` from - - :type logger: :class:`logging.Logger` - :param logger: The logger that needs to be set-up. - - :type argv: list of strings - :param argv: Command line arguments from caller. - """ - log_level = get_log_level(argv) - logger.setLevel(log_level) - stream_handler = logging.StreamHandler() - stream_handler.setLevel(log_level) - logger.addHandler(stream_handler) - - -setup_logger(LOGGER, sys.argv) diff --git a/gcloud_bigtable/_testing.py b/gcloud_bigtable/_testing.py deleted file mode 100644 index 46d9609..0000000 --- a/gcloud_bigtable/_testing.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Shared testing utilities.""" - - -class _MockCalled(object): - - def __init__(self, result=None): - self.called_args = [] - self.called_kwargs = [] - self.result = result - - def check_called(self, test_case, args_list, kwargs_list=None): - test_case.assertEqual(self.called_args, args_list) - if kwargs_list is None: - test_case.assertTrue(all([val == {} - for val in self.called_kwargs])) - else: - test_case.assertEqual(self.called_kwargs, kwargs_list) - - def __call__(self, *args, **kwargs): - self.called_args.append(args) - self.called_kwargs.append(kwargs) - return self.result - - -class _AttachedMethod(object): - - def __init__(self, parent, name): - self.parent = parent - self.name = name - - def __call__(self, *args, **kwargs): - self.parent._called.append((self.name, args, kwargs)) - curr_result = self.parent._results[0] - self.parent._results = self.parent._results[1:] - return curr_result - - -class _MockWithAttachedMethods(object): - - def __init__(self, *results): - self._results = results - self._called = [] - - def __getattr__(self, name): - # We need not worry about names: _results, _called - # since __getattribute__ will handle them. - return _AttachedMethod(self, name) - - -class _Monkey(object): - # context-manager for replacing module names in the scope of a test. - - def __init__(self, module, **kw): - self.module = module - self.to_restore = dict([(key, getattr(module, key)) for key in kw]) - for key, value in kw.items(): - setattr(module, key, value) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - for key, value in self.to_restore.items(): - setattr(self.module, key, value) diff --git a/gcloud_bigtable/client.py b/gcloud_bigtable/client.py deleted file mode 100644 index fef9300..0000000 --- a/gcloud_bigtable/client.py +++ /dev/null @@ -1,626 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Parent client for calling the Google Cloud Bigtable API. - -This is the base from which all interactions with the API occur. - -In the hierarchy of API concepts - -* a :class:`Client` owns a :class:`.Cluster` -* a :class:`.Cluster` owns a :class:`Table ` -* a :class:`Table ` owns a - :class:`ColumnFamily <.column_family.ColumnFamily>` -* a :class:`Table ` owns a :class:`Row <.row.Row>` - (and all the cells in the row) -""" - - -import copy -import os -import six -import socket - -from oauth2client.client import GoogleCredentials -from oauth2client.client import SignedJwtAssertionCredentials -from oauth2client.client import _get_application_default_credential_from_file - -try: - from google.appengine.api import app_identity -except ImportError: - app_identity = None - -from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._generated import bigtable_cluster_service_pb2 -from gcloud_bigtable._generated import bigtable_service_pb2 -from gcloud_bigtable._generated import bigtable_table_service_pb2 -from gcloud_bigtable._generated import operations_pb2 -from gcloud_bigtable._helpers import make_stub -from gcloud_bigtable.cluster import Cluster - - -TABLE_STUB_FACTORY = (bigtable_table_service_pb2. - early_adopter_create_BigtableTableService_stub) -TABLE_ADMIN_HOST = 'bigtabletableadmin.googleapis.com' -"""Table Admin API request host.""" -TABLE_ADMIN_PORT = 443 -"""Table Admin API request port.""" - -CLUSTER_STUB_FACTORY = (bigtable_cluster_service_pb2. - early_adopter_create_BigtableClusterService_stub) -CLUSTER_ADMIN_HOST = 'bigtableclusteradmin.googleapis.com' -"""Cluster Admin API request host.""" -CLUSTER_ADMIN_PORT = 443 -"""Cluster Admin API request port.""" - -DATA_STUB_FACTORY = (bigtable_service_pb2. - early_adopter_create_BigtableService_stub) -DATA_API_HOST = 'bigtable.googleapis.com' -"""Data API request host.""" -DATA_API_PORT = 443 -"""Data API request port.""" - -OPERATIONS_STUB_FACTORY = operations_pb2.early_adopter_create_Operations_stub - -ADMIN_SCOPE = 'https://www.googleapis.com/auth/cloud-bigtable.admin' -"""Scope for interacting with the Cluster Admin and Table Admin APIs.""" -DATA_SCOPE = 'https://www.googleapis.com/auth/cloud-bigtable.data' -"""Scope for reading and writing table data.""" -READ_ONLY_SCOPE = ('https://www.googleapis.com/auth/' - 'cloud-bigtable.data.readonly') -"""Scope for reading table data.""" - -PROJECT_ENV_VAR = 'GCLOUD_PROJECT' -"""Environment variable used to provide an implicit project ID.""" - -DEFAULT_TIMEOUT_SECONDS = 10 -"""The default timeout to use for API requests.""" - -DEFAULT_USER_AGENT = 'gcloud-bigtable-python' -"""The default user agent for API requests.""" - - -def _project_from_environment(): - """Attempts to get the project ID from an environment variable. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: The project ID provided or :data:`None` - """ - return os.getenv(PROJECT_ENV_VAR) - - -def _project_from_app_engine(): - """Gets the App Engine application ID if it can be inferred. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: App Engine application ID if running in App Engine, - else :data:`None`. - """ - if app_identity is None: - return None - - return app_identity.get_application_id() - - -def _project_from_compute_engine(): - """Gets the Compute Engine project ID if it can be inferred. - - Uses 169.254.169.254 for the metadata server to avoid request - latency from DNS lookup. - - See https://cloud.google.com/compute/docs/metadata#metadataserver - for information about this IP address. (This IP is also used for - Amazon EC2 instances, so the metadata flavor is crucial.) - - See https://github.com/google/oauth2client/issues/93 for context about - DNS latency. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: Compute Engine project ID if the metadata service is available, - else :data:`None`. - """ - host = '169.254.169.254' - uri_path = '/computeMetadata/v1/project/project-id' - headers = {'Metadata-Flavor': 'Google'} - connection = six.moves.http_client.HTTPConnection(host, timeout=0.1) - - try: - connection.request('GET', uri_path, headers=headers) - response = connection.getresponse() - if response.status == 200: - return response.read() - except socket.error: # socket.timeout or socket.error(64, 'Host is down') - pass - finally: - connection.close() - - -def _determine_project(project=None): - """Determine the project ID from the input or environment. - - When checking the environment, the following precedence is observed: - - * GCLOUD_PROJECT environment variable - * Google App Engine application ID - * Google Compute Engine project ID (from metadata server) - - :type project: str - :param project: (Optional) The ID of the project which owns the - clusters, tables and data. If not provided, will attempt - to determine from the environment. - - :rtype: str - :returns: The project ID provided or inferred from the environment. - :raises: :class:`EnvironmentError` if the project ID was not - passed in and can't be inferred from the environment. - """ - if project is None: - project = _project_from_environment() - - if project is None: - project = _project_from_app_engine() - - if project is None: - project = _project_from_compute_engine() - - if project is None: - raise EnvironmentError('Project ID was not provided and could not ' - 'be determined from environment.') - - return project - - -class Client(object): - """Client for interacting with Google Cloud Bigtable API. - - .. note:: - - Since the Cloud Bigtable API requires the gRPC transport, no - ``http`` argument is accepted by this class. - - :type project: :class:`str` or :func:`unicode ` - :param project: (Optional) The ID of the project which owns the - clusters, tables and data. If not provided, will - attempt to determine from the environment. - - :type credentials: - :class:`OAuth2Credentials ` or - :data:`NoneType ` - :param credentials: (Optional) The OAuth2 Credentials to use for this - cluster. If not provided, defaulst to the Google - Application Default Credentials. - - :type read_only: bool - :param read_only: (Optional) Boolean indicating if the data scope should be - for reading only (or for writing as well). Defaults to - :data:`False`. - - :type admin: bool - :param admin: (Optional) Boolean indicating if the client will be used to - interact with the Cluster Admin or Table Admin APIs. This - requires the :const:`ADMIN_SCOPE`. Defaults to :data:`False`. - - :type user_agent: str - :param user_agent: (Optional) The user agent to be used with API request. - Defaults to :const:`DEFAULT_USER_AGENT`. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. If not - passed, defaults to - :const:`DEFAULT_TIMEOUT_SECONDS`. - - :raises: :class:`ValueError ` if both ``read_only`` - and ``admin`` are :data:`True` - """ - - def __init__(self, project=None, credentials=None, - read_only=False, admin=False, user_agent=DEFAULT_USER_AGENT, - timeout_seconds=DEFAULT_TIMEOUT_SECONDS): - self.project = _determine_project(project) - if credentials is None: - credentials = GoogleCredentials.get_application_default() - - if read_only and admin: - raise ValueError('A read-only client cannot also perform' - 'administrative actions.') - - scopes = [] - if read_only: - scopes.append(READ_ONLY_SCOPE) - else: - scopes.append(DATA_SCOPE) - - if admin: - scopes.append(ADMIN_SCOPE) - - self._admin = bool(admin) - self._credentials = credentials.create_scoped(scopes) - self.user_agent = user_agent - self.timeout_seconds = timeout_seconds - - # These will be set in start(). - self._data_stub_internal = None - self._cluster_stub_internal = None - self._operations_stub_internal = None - self._table_stub_internal = None - - @classmethod - def from_service_account_json(cls, json_credentials_path, project=None, - read_only=False, admin=False): - """Factory to retrieve JSON credentials while creating client object. - - :type json_credentials_path: str - :param json_credentials_path: The path to a private key file (this file - was given to you when you created the - service account). This file must contain - a JSON object with a private key and - other credentials information (downloaded - from the Google APIs console). - - :type project: str - :param project: The ID of the project which owns the clusters, - tables and data. Will be passed to :class:`Client` - constructor. - - :type read_only: bool - :param read_only: Boolean indicating if the data scope should be - for reading only (or for writing as well). Will be - passed to :class:`Client` constructor. - - :type admin: bool - :param admin: Boolean indicating if the client will be used to - interact with the Cluster Admin or Table Admin APIs. Will - be passed to :class:`Client` constructor. - - :rtype: :class:`Client` - :returns: The client created with the retrieved JSON credentials. - """ - credentials = _get_application_default_credential_from_file( - json_credentials_path) - return cls(credentials=credentials, project=project, - read_only=read_only, admin=admin) - - @classmethod - def from_service_account_p12(cls, client_email, private_key_path, - project=None, read_only=False, - admin=False): - """Factory to retrieve P12 credentials while creating client object. - - .. note:: - Unless you have an explicit reason to use a PKCS12 key for your - service account, we recommend using a JSON key. - - :type client_email: str - :param client_email: The e-mail attached to the service account. - - :type private_key_path: str - :param private_key_path: The path to a private key file (this file was - given to you when you created the service - account). This file must be in P12 format. - - :type project: str - :param project: The ID of the project which owns the clusters, - tables and data. Will be passed to :class:`Client` - constructor. - - :type read_only: bool - :param read_only: Boolean indicating if the data scope should be - for reading only (or for writing as well). Will be - passed to :class:`Client` constructor. - - :type admin: bool - :param admin: Boolean indicating if the client will be used to - interact with the Cluster Admin or Table Admin APIs. Will - be passed to :class:`Client` constructor. - - :rtype: :class:`Client` - :returns: The client created with the retrieved P12 credentials. - """ - credentials = SignedJwtAssertionCredentials( - service_account_name=client_email, - private_key=_get_contents(private_key_path)) - return cls(credentials=credentials, project=project, - read_only=read_only, admin=admin) - - def copy(self): - """Make a copy of this client. - - Copies the local data stored as simple types but does not copy the - current state of any open connections with the Cloud Bigtable API. - - :rtype: :class:`.Client` - :returns: A copy of the current client. - """ - copied_creds = copy.deepcopy(self._credentials) - return self.__class__( - self.project, - copied_creds, - READ_ONLY_SCOPE in copied_creds.scopes, - self._admin, - self.user_agent, - self.timeout_seconds, - ) - - @property - def credentials(self): - """Getter for client's credentials. - - :rtype: - :class:`OAuth2Credentials ` - :returns: The credentials stored on the client. - """ - return self._credentials - - @property - def project_name(self): - """Project name to be used with Cluster Admin API. - - .. note:: - This property will not change if ``project`` does not, but the - return value is not cached. - - The project name is of the form - - ``"projects/{project}"`` - - :rtype: str - :returns: The project name to be used with the Cloud Bigtable Admin - API RPC service. - """ - return 'projects/' + self.project - - @property - def _data_stub(self): - """Getter for the gRPC stub used for the Data API. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client has not been :meth:`start`-ed. - """ - if self._data_stub_internal is None: - raise ValueError('Client has not been started.') - return self._data_stub_internal - - @property - def _cluster_stub(self): - """Getter for the gRPC stub used for the Cluster Admin API. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client is not an admin client or if it has not been - :meth:`start`-ed. - """ - if not self._admin: - raise ValueError('Client is not an admin client.') - if self._cluster_stub_internal is None: - raise ValueError('Client has not been started.') - return self._cluster_stub_internal - - @property - def _operations_stub(self): - """Getter for the gRPC stub used for the Operations API. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client is not an admin client or if it has not been - :meth:`start`-ed. - """ - if not self._admin: - raise ValueError('Client is not an admin client.') - if self._operations_stub_internal is None: - raise ValueError('Client has not been started.') - return self._operations_stub_internal - - @property - def _table_stub(self): - """Getter for the gRPC stub used for the Table Admin API. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - :raises: :class:`ValueError ` if the current - client is not an admin client or if it has not been - :meth:`start`-ed. - """ - if not self._admin: - raise ValueError('Client is not an admin client.') - if self._table_stub_internal is None: - raise ValueError('Client has not been started.') - return self._table_stub_internal - - def _make_data_stub(self): - """Creates gRPC stub to make requests to the Data API. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - """ - return make_stub(self, DATA_STUB_FACTORY, - DATA_API_HOST, DATA_API_PORT) - - def _make_cluster_stub(self): - """Creates gRPC stub to make requests to the Cluster Admin API. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - """ - return make_stub(self, CLUSTER_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) - - def _make_operations_stub(self): - """Creates gRPC stub to make requests to the Operations API. - - These are for long-running operations of the Cluster Admin API, - hence the host and port matching. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - """ - return make_stub(self, OPERATIONS_STUB_FACTORY, - CLUSTER_ADMIN_HOST, CLUSTER_ADMIN_PORT) - - def _make_table_stub(self): - """Creates gRPC stub to make requests to the Table Admin API. - - :rtype: :class:`grpc.early_adopter.implementations._Stub` - :returns: A gRPC stub object. - """ - return make_stub(self, TABLE_STUB_FACTORY, - TABLE_ADMIN_HOST, TABLE_ADMIN_PORT) - - def is_started(self): - """Check if the client has been started. - - :rtype: bool - :returns: Boolean indicating if the client has been started. - """ - return self._data_stub_internal is not None - - def start(self): - """Prepare the client to make requests. - - Activates gRPC contexts for making requests to the Bigtable - Service(s). - """ - if self.is_started(): - return - - # NOTE: We __enter__ the stubs more-or-less permanently. This is - # because only after entering the context managers is the - # connection created. We don't want to immediately close - # those connections since the client will make many - # requests with it over HTTP/2. - self._data_stub_internal = self._make_data_stub() - self._data_stub_internal.__enter__() - if self._admin: - self._cluster_stub_internal = self._make_cluster_stub() - self._operations_stub_internal = self._make_operations_stub() - self._table_stub_internal = self._make_table_stub() - - self._cluster_stub_internal.__enter__() - self._operations_stub_internal.__enter__() - self._table_stub_internal.__enter__() - - def stop(self): - """Closes all the open gRPC clients.""" - if not self.is_started(): - return - - # When exit-ing, we pass None as the exception type, value and - # traceback to __exit__. - self._data_stub_internal.__exit__(None, None, None) - if self._admin: - self._cluster_stub_internal.__exit__(None, None, None) - self._operations_stub_internal.__exit__(None, None, None) - self._table_stub_internal.__exit__(None, None, None) - - self._data_stub_internal = None - self._cluster_stub_internal = None - self._operations_stub_internal = None - self._table_stub_internal = None - - def cluster(self, zone, cluster_id, display_name=None, serve_nodes=3): - """Factory to create a cluster associated with this client. - - :type zone: str - :param zone: The name of the zone where the cluster resides. - - :type cluster_id: str - :param cluster_id: The ID of the cluster. - - :type display_name: str - :param display_name: (Optional) The display name for the cluster in the - Cloud Console UI. (Must be between 4 and 30 - characters.) If this value is not set in the - constructor, will fall back to the cluster ID. - - :type serve_nodes: int - :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3. - - :rtype: :class:`.Cluster` - :returns: The cluster owned by this client. - """ - return Cluster(zone, cluster_id, self, - display_name=display_name, serve_nodes=serve_nodes) - - def list_zones(self, timeout_seconds=None): - """Lists zones associated with project. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on client. - - :rtype: list - :returns: The names (as :class:`str`) of the zones - :raises: :class:`ValueError ` if one of the - zones is not in ``OK`` state. - """ - request_pb = messages_pb2.ListZonesRequest(name=self.project_name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self._cluster_stub.ListZones.async(request_pb, - timeout_seconds) - # We expect a `.messages_pb2.ListZonesResponse` - list_zones_response = response.result() - - result = [] - for zone in list_zones_response.zones: - if zone.status != data_pb2.Zone.OK: - raise ValueError('Zone %s not in OK state' % ( - zone.display_name,)) - result.append(zone.display_name) - return result - - def list_clusters(self, timeout_seconds=None): - """Lists clusters owned by the project. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on client. - - :rtype: tuple - :returns: A pair of results, the first is a list of :class:`.Cluster` s - returned and the second is a list of strings (the failed - zones in the request). - """ - request_pb = messages_pb2.ListClustersRequest(name=self.project_name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self._cluster_stub.ListClusters.async(request_pb, - timeout_seconds) - # We expect a `.messages_pb2.ListClustersResponse` - list_clusters_response = response.result() - - failed_zones = [zone.display_name - for zone in list_clusters_response.failed_zones] - clusters = [Cluster.from_pb(cluster_pb, self) - for cluster_pb in list_clusters_response.clusters] - return clusters, failed_zones - - -def _get_contents(filename): - """Get the contents of a file. - - This is just implemented so we can stub out while testing. - - :type filename: :class:`str` or :func:`unicode ` - :param filename: The name of a file to open. - - :rtype: bytes - :returns: The bytes loaded from the file. - """ - with open(filename, 'rb') as file_obj: - return file_obj.read() diff --git a/gcloud_bigtable/cluster.py b/gcloud_bigtable/cluster.py deleted file mode 100644 index 00a9c72..0000000 --- a/gcloud_bigtable/cluster.py +++ /dev/null @@ -1,441 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Cluster.""" - - -import re - -from gcloud_bigtable._generated import bigtable_cluster_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as table_messages_pb2) -from gcloud_bigtable._generated import operations_pb2 -from gcloud_bigtable._helpers import _parse_pb_any_to_native -from gcloud_bigtable._helpers import _pb_timestamp_to_datetime -from gcloud_bigtable._helpers import _require_pb_property -from gcloud_bigtable.table import Table - - -_CLUSTER_NAME_RE = re.compile(r'^projects/(?P[^/]+)/' - r'zones/(?P[^/]+)/clusters/' - r'(?P[a-z][-a-z0-9]*)$') -_OPERATION_NAME_RE = re.compile(r'^operations/projects/([^/]+)/zones/([^/]+)/' - r'clusters/([a-z][-a-z0-9]*)/operations/' - r'(?P\d+)$') - - -def _prepare_create_request(cluster): - """Creates a protobuf request for a CreateCluster request. - - :type cluster: :class:`Cluster` - :param cluster: The cluster to be created. - - :rtype: :class:`.messages_pb2.CreateClusterRequest` - :returns: The CreateCluster request object containing the cluster info. - """ - zone_full_name = ('projects/' + cluster.project + - '/zones/' + cluster.zone) - return messages_pb2.CreateClusterRequest( - name=zone_full_name, - cluster_id=cluster.cluster_id, - cluster=data_pb2.Cluster( - display_name=cluster.display_name, - serve_nodes=cluster.serve_nodes, - ), - ) - - -def _process_operation(operation_pb): - """Processes a create protobuf response. - - :type operation_pb: :class:`operations_pb2.Operation` - :param operation_pb: The long-running operation response from a - Create/Update/Undelete cluster request. - - :rtype: tuple - :returns: A pair of an integer and datetime stamp. The integer is the ID - of the operation (``operation_id``) and the timestamp when - the create operation began (``operation_begin``). - :raises: :class:`ValueError ` if the operation name - doesn't match the :data:`_OPERATION_NAME_RE` regex. - """ - match = _OPERATION_NAME_RE.match(operation_pb.name) - if match is None: - raise ValueError('Cluster create operation name was not in the ' - 'expected format.', operation_pb.name) - operation_id = int(match.group('operation_id')) - - request_metadata = _parse_pb_any_to_native(operation_pb.metadata) - operation_begin = _pb_timestamp_to_datetime( - request_metadata.request_time) - - return operation_id, operation_begin - - -class Cluster(object): - """Representation of a Google Cloud Bigtable Cluster. - - We can use a :class:`Cluster` to: - - * :meth:`reload` itself - * :meth:`create` itself - * Check if an :meth:`operation_finished` (each of :meth:`create`, - :meth:`update` and :meth:`undelete` return with long-running operations) - * :meth:`update` itself - * :meth:`delete` itself - * :meth:`undelete` itself - - .. note:: - - For now, we leave out the ``default_storage_type`` (an enum) - which if not sent will end up as :data:`.data_pb2.STORAGE_SSD`. - - :type zone: str - :param zone: The name of the zone where the cluster resides. - - :type cluster_id: str - :param cluster_id: The ID of the cluster. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. Provides - authorization and a project ID. - - :type display_name: str - :param display_name: (Optional) The display name for the cluster in the - Cloud Console UI. (Must be between 4 and 30 - characters.) If this value is not set in the - constructor, will fall back to the cluster ID. - - :type serve_nodes: int - :param serve_nodes: (Optional) The number of nodes in the cluster. - Defaults to 3. - """ - - def __init__(self, zone, cluster_id, client, - display_name=None, serve_nodes=3): - self.zone = zone - self.cluster_id = cluster_id - self.display_name = display_name or cluster_id - self.serve_nodes = serve_nodes - self._client = client - self._operation_type = None - self._operation_id = None - self._operation_begin = None - - def _update_from_pb(self, cluster_pb): - self.display_name = _require_pb_property( - cluster_pb, 'display_name', None) - self.serve_nodes = _require_pb_property( - cluster_pb, 'serve_nodes', None) - - @classmethod - def from_pb(cls, cluster_pb, client): - """Creates a cluster instance from a protobuf. - - :type cluster_pb: :class:`bigtable_cluster_data_pb2.Cluster` - :param cluster_pb: A cluster protobuf object. - - :type client: :class:`.client.Client` - :param client: The client that owns the cluster. - - :rtype: :class:`Cluster` - :returns: The cluster parsed from the protobuf response. - :raises: :class:`ValueError ` if the cluster - name does not match :data:`_CLUSTER_NAME_RE` or if the parsed - project ID does not match the project ID on the client. - """ - match = _CLUSTER_NAME_RE.match(cluster_pb.name) - if match is None: - raise ValueError('Cluster protobuf name was not in the ' - 'expected format.', cluster_pb.name) - if match.group('project') != client.project: - raise ValueError('Project ID on cluster does not match the ' - 'project ID on the client') - - result = cls(match.group('zone'), match.group('cluster_id'), client) - result._update_from_pb(cluster_pb) - return result - - def copy(self): - """Make a copy of this cluster. - - Copies the local data stored as simple types but does not copy the - current state of any operations with the Cloud Bigtable API. Also - copies the client attached to this instance. - - :rtype: :class:`.Cluster` - :returns: A copy of the current cluster. - """ - new_client = self.client.copy() - return Cluster(self.zone, self.cluster_id, new_client, - display_name=self.display_name, - serve_nodes=self.serve_nodes) - - @property - def client(self): - """Getter for cluster's client. - - :rtype: :class:`.client.Client` - :returns: The client stored on the cluster. - """ - return self._client - - @property - def project(self): - """Getter for cluster's project ID. - - :rtype: str - :returns: The project ID for the cluster (is stored on the client). - """ - return self._client.project - - @property - def timeout_seconds(self): - """Getter for cluster's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default stored on the cluster's client. - """ - return self._client.timeout_seconds - - @property - def name(self): - """Cluster name used in requests. - - .. note:: - This property will not change if ``zone`` and ``cluster_id`` do not, - but the return value is not cached. - - The cluster name is of the form - - ``"projects/{project}/zones/{zone}/clusters/{cluster_id}"`` - - :rtype: str - :returns: The cluster name. - """ - return (self.client.project_name + '/zones/' + self.zone + - '/clusters/' + self.cluster_id) - - def table(self, table_id): - """Factory to create a table associated with this cluster. - - :type table_id: str - :param table_id: The ID of the table. - - :rtype: :class:`Table ` - :returns: The table owned by this cluster. - """ - return Table(table_id, self) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - # NOTE: This does not compare the configuration values, such as - # the serve_nodes or display_name. This is intentional, since - # the same cluster can be in different states if not - # synchronized. This suggests we should use `project` - # instead of `client` for the third comparison. - return (other.zone == self.zone and - other.cluster_id == self.cluster_id and - other.client == self.client) - - def __ne__(self, other): - return not self.__eq__(other) - - def reload(self, timeout_seconds=None): - """Reload the metadata for this cluster. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - """ - request_pb = messages_pb2.GetClusterRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.GetCluster.async(request_pb, - timeout_seconds) - # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = response.result() - - # NOTE: _update_from_pb does not check that the project, zone and - # cluster ID on the response match the request. - self._update_from_pb(cluster_pb) - - def operation_finished(self, timeout_seconds=None): - """Check if the current operation has finished. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - - :rtype: bool - :returns: A boolean indicating if the current operation has completed. - :raises: :class:`ValueError ` if there is no - current operation set. - """ - if self._operation_id is None: - raise ValueError('There is no current operation.') - - operation_name = ('operations/' + self.name + - '/operations/%d' % (self._operation_id,)) - request_pb = operations_pb2.GetOperationRequest(name=operation_name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._operations_stub.GetOperation.async( - request_pb, timeout_seconds) - # We expact a `._generated.operations_pb2.Operation`. - operation_pb = response.result() - - if operation_pb.done: - self._operation_type = None - self._operation_id = None - self._operation_begin = None - return True - else: - return False - - def create(self, timeout_seconds=None): - """Create this cluster. - - .. note:: - - Uses the ``project``, ``zone`` and ``cluster_id`` on the current - :class:`Cluster` in addition to the ``display_name`` and - ``serve_nodes``. If you'd like to change them before creating, - reset the values via - - .. code:: python - - cluster.display_name = 'New display name' - cluster.cluster_id = 'i-changed-my-mind' - - before calling :meth:`create`. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - """ - request_pb = _prepare_create_request(self) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.CreateCluster.async( - request_pb, timeout_seconds) - # We expect an `operations_pb2.Operation`. - cluster_pb = response.result() - - self._operation_type = 'create' - self._operation_id, self._operation_begin = _process_operation( - cluster_pb.current_operation) - - def update(self, timeout_seconds=None): - """Update this cluster. - - .. note:: - - Updates the ``display_name`` and ``serve_nodes``. If you'd like to - change them before updating, reset the values via - - .. code:: python - - cluster.display_name = 'New display name' - cluster.serve_nodes = 3 - - before calling :meth:`update`. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - """ - request_pb = data_pb2.Cluster( - name=self.name, - display_name=self.display_name, - serve_nodes=self.serve_nodes, - ) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.UpdateCluster.async( - request_pb, timeout_seconds) - # We expect a `._generated.bigtable_cluster_data_pb2.Cluster`. - cluster_pb = response.result() - - self._operation_type = 'update' - self._operation_id, self._operation_begin = _process_operation( - cluster_pb.current_operation) - - def delete(self, timeout_seconds=None): - """Delete this cluster. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - """ - request_pb = messages_pb2.DeleteClusterRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.DeleteCluster.async( - request_pb, timeout_seconds) - # We expect a `._generated.empty_pb2.Empty` - response.result() - - def undelete(self, timeout_seconds=None): - """Undelete this cluster. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - """ - request_pb = messages_pb2.UndeleteClusterRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._cluster_stub.UndeleteCluster.async( - request_pb, timeout_seconds) - # We expect a `._generated.operations_pb2.Operation` - operation_pb2 = response.result() - - self._operation_type = 'undelete' - self._operation_id, self._operation_begin = _process_operation( - operation_pb2) - - def list_tables(self, timeout_seconds=None): - """List the tables in this cluster. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - cluster. - - :rtype: list of :class:`Table ` - :returns: The list of tables owned by the cluster. - :raises: :class:`ValueError ` if one of the - returned tables has a name that is not of the expected format. - """ - request_pb = table_messages_pb2.ListTablesRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.ListTables.async(request_pb, - timeout_seconds) - # We expect a `table_messages_pb2.ListTablesResponse` - table_list_pb = response.result() - - result = [] - for table_pb in table_list_pb.tables: - before, table_id = table_pb.name.split( - self.name + '/tables/', 1) - if before != '': - raise ValueError('Table name %s not of expected format' % ( - table_pb.name,)) - result.append(self.table(table_id)) - - return result diff --git a/gcloud_bigtable/column_family.py b/gcloud_bigtable/column_family.py deleted file mode 100644 index 58b3973..0000000 --- a/gcloud_bigtable/column_family.py +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Column Family.""" - - -from gcloud_bigtable._generated import bigtable_table_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._helpers import _duration_pb_to_timedelta -from gcloud_bigtable._helpers import _timedelta_to_duration_pb - - -class GarbageCollectionRule(object): - """Table garbage collection rule. - - Cells in the table fitting the rule will be deleted during - garbage collection. - - These values can be combined via :class:`GarbageCollectionRuleUnion` and - :class:`GarbageCollectionRuleIntersection`. - - .. note:: - - At most one of ``max_num_versions`` and ``max_age`` can be specified - at once. - - .. note:: - - A string ``gc_expression`` can also be used with API requests, but - that value would be superceded by a ``gc_rule``. As a result, we - don't support that feature and instead support via this native - object. - - :type max_num_versions: int - :param max_num_versions: The maximum number of versions - - :type max_age: :class:`datetime.timedelta` - :param max_age: The maximum age allowed for a cell in the table. - - :raises: :class:`TypeError ` if both - ``max_num_versions`` and ``max_age`` are set. - """ - - def __init__(self, max_num_versions=None, max_age=None): - self.max_num_versions = max_num_versions - self.max_age = max_age - self._check_single_value() - - def _check_single_value(self): - """Checks that at most one value is set on the instance. - - :raises: :class:`TypeError ` if not exactly one - value set on the instance. - """ - if self.max_num_versions is not None and self.max_age is not None: - raise TypeError('At most one of max_num_versions and ' - 'max_age can be set') - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.max_num_versions == self.max_num_versions and - other.max_age == self.max_age) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`GarbageCollectionRule` to a protobuf. - - :rtype: :class:`.data_pb2.GcRule` - :returns: The converted current object. - """ - self._check_single_value() - gc_rule_kwargs = {} - if self.max_num_versions is not None: - gc_rule_kwargs['max_num_versions'] = self.max_num_versions - if self.max_age is not None: - gc_rule_kwargs['max_age'] = _timedelta_to_duration_pb(self.max_age) - return data_pb2.GcRule(**gc_rule_kwargs) - - -class GarbageCollectionRuleUnion(object): - """Union of garbage collection rules. - - :type rules: list - :param rules: List of :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion` and/or - :class:`GarbageCollectionRuleIntersection` - """ - - def __init__(self, rules=None): - self.rules = rules - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.rules == self.rules - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the union into a single gc rule as a protobuf. - - :rtype: :class:`.data_pb2.GcRule` - :returns: The converted current object. - """ - union = data_pb2.GcRule.Union( - rules=[rule.to_pb() for rule in self.rules]) - return data_pb2.GcRule(union=union) - - -class GarbageCollectionRuleIntersection(object): - """Intersection of garbage collection rules. - - :type rules: list - :param rules: List of :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion` and/or - :class:`GarbageCollectionRuleIntersection` - """ - - def __init__(self, rules=None): - self.rules = rules - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.rules == self.rules - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the intersection into a single gc rule as a protobuf. - - :rtype: :class:`.data_pb2.GcRule` - :returns: The converted current object. - """ - intersection = data_pb2.GcRule.Intersection( - rules=[rule.to_pb() for rule in self.rules]) - return data_pb2.GcRule(intersection=intersection) - - -class ColumnFamily(object): - """Representation of a Google Cloud Bigtable Column Family. - - We can use a :class:`ColumnFamily` to: - - * :meth:`create` itself - * :meth:`update` itself - * :meth:`delete` itself - - :type column_family_id: str - :param column_family_id: The ID of the column family. Must be of the - form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type table: :class:`Table ` - :param table: The table that owns the column family. - - :type gc_rule: :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion` or - :class:`GarbageCollectionRuleIntersection` - :param gc_rule: (Optional) The garbage collection settings for this - column family. - """ - - def __init__(self, column_family_id, table, gc_rule=None): - self.column_family_id = column_family_id - self._table = table - self.gc_rule = gc_rule - - @property - def table(self): - """Getter for column family's table. - - :rtype: :class:`Table ` - :returns: The table stored on the column family. - """ - return self._table - - @property - def client(self): - """Getter for column family's client. - - :rtype: :class:`.client.Client` - :returns: The client that owns this column family. - """ - return self.table.client - - @property - def timeout_seconds(self): - """Getter for column family's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default. - """ - return self.table.timeout_seconds - - @property - def name(self): - """Column family name used in requests. - - .. note:: - - This property will not change if ``column_family_id`` does not, but - the return value is not cached. - - The table name is of the form - - ``"projects/../zones/../clusters/../tables/../columnFamilies/.."`` - - :rtype: str - :returns: The column family name. - """ - return self.table.name + '/columnFamilies/' + self.column_family_id - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.column_family_id == self.column_family_id and - other.gc_rule == self.gc_rule and - other.table == self.table) - - def __ne__(self, other): - return not self.__eq__(other) - - def create(self, timeout_seconds=None): - """Create this column family. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - column family. - """ - if self.gc_rule is None: - column_family = data_pb2.ColumnFamily() - else: - column_family = data_pb2.ColumnFamily(gc_rule=self.gc_rule.to_pb()) - request_pb = messages_pb2.CreateColumnFamilyRequest( - name=self.table.name, - column_family_id=self.column_family_id, - column_family=column_family, - ) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.CreateColumnFamily.async( - request_pb, timeout_seconds) - # We expect a `.data_pb2.ColumnFamily` - response.result() - - def update(self, timeout_seconds=None): - """Update this column family. - - .. note:: - - The Bigtable Table Admin API currently returns - - ``BigtableTableService.UpdateColumnFamily is not yet implemented`` - - when this method is used. It's unclear when this method will - actually be supported by the API. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - column family. - """ - request_kwargs = {'name': self.name} - if self.gc_rule is not None: - request_kwargs['gc_rule'] = self.gc_rule.to_pb() - request_pb = data_pb2.ColumnFamily(**request_kwargs) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.UpdateColumnFamily.async( - request_pb, timeout_seconds) - # We expect a `.data_pb2.ColumnFamily` - response.result() - - def delete(self, timeout_seconds=None): - """Delete this column family. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on - column family. - """ - request_pb = messages_pb2.DeleteColumnFamilyRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.DeleteColumnFamily.async( - request_pb, timeout_seconds) - # We expect a `._generated.empty_pb2.Empty` - response.result() - - -def _gc_rule_from_pb(gc_rule_pb): - """Convert a protobuf GC rule to a Python version. - - :type gc_rule_pb: :class:`.data_pb2.GcRule` - :param gc_rule_pb: The GC rule to convert. - - :rtype: :class:`GarbageCollectionRule`, - :class:`GarbageCollectionRuleUnion`, - :class:`GarbageCollectionRuleIntersection` or - :data:`NoneType ` - :returns: An instance of one of the native rules defined - in :module:`column_family` or :data:`None` if no values were - set on the protobuf passed in. - :raises: :class:`ValueError ` if more than one - property has been set on the GC rule. - """ - all_fields = [field.name for field in gc_rule_pb._fields] - if len(all_fields) == 0: - return None - elif len(all_fields) > 1: - raise ValueError('At most one field can be set on a GC rule.') - - field_name = all_fields[0] - if field_name == 'max_num_versions': - return GarbageCollectionRule( - max_num_versions=gc_rule_pb.max_num_versions) - elif field_name == 'max_age': - max_age = _duration_pb_to_timedelta(gc_rule_pb.max_age) - return GarbageCollectionRule(max_age=max_age) - elif field_name == 'union': - all_rules = gc_rule_pb.union.rules - return GarbageCollectionRuleUnion( - rules=[_gc_rule_from_pb(rule) for rule in all_rules]) - elif field_name == 'intersection': - all_rules = gc_rule_pb.intersection.rules - return GarbageCollectionRuleIntersection( - rules=[_gc_rule_from_pb(rule) for rule in all_rules]) diff --git a/gcloud_bigtable/happybase/__init__.py b/gcloud_bigtable/happybase/__init__.py deleted file mode 100644 index eb274a3..0000000 --- a/gcloud_bigtable/happybase/__init__.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase package. - -This package is intended to emulate the HappyBase library using -Google Cloud Bigtable as the backing store. - -Differences in Public API -------------------------- - -Some concepts from HBase/Thrift do not map directly to the Cloud -Bigtable API. As a result, the following instance methods and functions -could not be implemented: - -* :meth:`.Connection.enable_table` - no concept of enabled/disabled -* :meth:`.Connection.disable_table` - no concept of enabled/disabled -* :meth:`.Connection.is_table_enabled` - no concept of enabled/disabled -* :meth:`.Connection.compact_table` - table storage is opaque to user -* :func:`make_row() ` - helper - needed for Thrift library -* :func:`make_ordered_row() ` - - helper needed for Thrift library -* :meth:`Table.regions() ` - - tables in Cloud Bigtable do not expose internal storage details -* :meth:`Table.counter_set() \ - ` - method can't - be atomic, so we disable it -* The ``__version__`` value for the HappyBase package is :data:`None` - -In addition, many of the constants from :mod:`.connection` are specific -to HBase and are defined as :data:`None` in our module: - -* ``COMPAT_MODES`` -* ``THRIFT_TRANSPORTS`` -* ``THRIFT_PROTOCOLS`` -* ``DEFAULT_HOST`` -* ``DEFAULT_PORT`` -* ``DEFAULT_TRANSPORT`` -* ``DEFAULT_COMPAT`` -* ``DEFAULT_PROTOCOL`` - -Two of these ``DEFAULT_HOST`` and ``DEFAULT_PORT``, are even imported in -the main ``happybase`` package. - -Finally, we do not provide the ``util`` module. Though it is public in the -HappyBase library, it provides no core functionality. - -API Behavior Changes --------------------- - -* Since there is no concept of an enabled / disabled table, calling - :meth:`.Connection.delete_table` with ``disable=True`` can't be supported. - Using that argument will result in a - :class:`ValueError `. -* The :class:`.Connection` constructor **disables** the use of several - arguments and will throw a :class:`ValueError ` if - any of them are passed in as keyword arguments. The arguments are: - - * ``host`` - * ``port`` - * ``compat`` - * ``transport`` - * ``protocol`` -* In order to make :class:`.Connection` compatible with Cloud Bigtable, we - add a ``cluster`` keyword argument to allow user's to pass in their own - :class:`.Cluster` (which they can construct beforehand). - - For example: - - .. code:: python - - from gcloud_bigtable.client import Client - client = Client(project=PROJECT_ID, admin=True) - cluster = client.cluster(zone, cluster_id) - cluster.reload() - - from gcloud_bigtable.happybase import Connection - connection = Connection(cluster=cluster) - -* Any uses of the ``wal`` (Write Ahead Log) argument will result in a - :class:`ValueError ` as well. This includes - uses in: - - * :class:`.Batch` constructor - * :meth:`.Batch.put` - * :meth:`.Batch.delete` - * :meth:`Table.put() ` - * :meth:`Table.delete() ` - * :meth:`Table.batch() ` factory -* When calling :meth:`.Connection.create_table`, the majority of HBase column - family options cannot be used. Among - - * ``max_versions`` - * ``compression`` - * ``in_memory`` - * ``bloom_filter_type`` - * ``bloom_filter_vector_size`` - * ``bloom_filter_nb_hashes`` - * ``block_cache_enabled`` - * ``time_to_live`` - - Only ``max_versions`` and ``time_to_live`` are availabe in Cloud Bigtable - (as ``max_num_versions`` and ``max_age``). - - In addition to using a dictionary for specifying column family options, - we also accept instances of :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleUnion` or - :class:`.GarbageCollectionRuleIntersection`. -* The ``batch_size`` attribute in :class:`.Batch` cannot be truly mapped - from HBase to Cloud Bigtable. The main reason for this is that the Cloud - Bigtable API can only send mutations for a single row (via ``MutateRow``, - ``CheckAndMutateRow``, and ``ReadModifyWriteRow``) whereas HBase sends - all mutations at once. This requires a single request to be sent for each - mutated row in the batch. This should not be noticeable since gRPC - uses HTTP/2. However, some of the requests may fail part way through and - the process of applying all mutations cannot be rolled back. -* :meth:`Table.scan() ` no longer - accepts the following arguments (which will result in a - :class:`ValueError `): - - * ``batch_size`` - * ``scan_batching`` - * ``sorted_columns`` - -* Using a HBase filter string in - :meth:`Table.scan() ` is - not possible with Cloud Bigtable and will result in a - :class:`TypeError `. However, the method now accepts - :class:`.RowFilter` instances (and related classes). -* :meth:`.Batch.delete` (and hence - :meth:`Table.delete() `) - will fail when either a row or column family delete is attempted with - a ``timestamp``. This is because the Cloud Bigtable API uses the - ``DeleteFromFamily`` and ``DeleteFromRow`` mutations for these deletes, and - neither of these mutations support a timestamp. -""" - -from gcloud_bigtable.happybase.batch import Batch -from gcloud_bigtable.happybase.connection import Connection -from gcloud_bigtable.happybase.connection import DEFAULT_HOST -from gcloud_bigtable.happybase.connection import DEFAULT_PORT -from gcloud_bigtable.happybase.pool import ConnectionPool -from gcloud_bigtable.happybase.pool import NoConnectionsAvailable -from gcloud_bigtable.happybase.table import Table - - -# Values from HappyBase that we don't reproduce / are not relevant. -__version__ = None diff --git a/gcloud_bigtable/happybase/batch.py b/gcloud_bigtable/happybase/batch.py deleted file mode 100644 index 3b006ba..0000000 --- a/gcloud_bigtable/happybase/batch.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase batch module.""" - - -import datetime -import six - -from gcloud_bigtable._helpers import _microseconds_to_timestamp -from gcloud_bigtable.row import TimestampRange - - -_WAL_SENTINEL = object() -# Assumed granularity of timestamps in Cloud Bigtable. -_ONE_MILLISECOND = datetime.timedelta(microseconds=1000) - - -def _get_column_pairs(columns, require_qualifier=False): - """Turns a list of column or column families in parsed pairs. - - Turns a column family (``fam`` or ``fam:``) into a pair such - as ``['fam', None]`` and turns a column (``fam:col``) into - ``['fam', 'col']``. - - :type columns: list - :param columns: Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type require_qualifier: bool - :param require_qualifier: Boolean indicating if the columns should - all have a qualifier or not. - - :rtype: list - :returns: List of pairs, where the first element in each pair is the - column family and the second is the column qualifier - (or :data:`None`). - :raises: :class:`ValueError ` if any of the columns - are not of the expected format. - :class:`ValueError ` if - ``require_qualifier`` is :data:`True` and one of the values is - for an entire column family - """ - column_pairs = [] - for column in columns: - # Remove trailing colons (i.e. for standalone column family). - column = column.rstrip(':') - num_colons = column.count(':') - if num_colons == 0: - # column is a column family. - if require_qualifier: - raise ValueError('column does not contain a qualifier', - column) - else: - column_pairs.append([column, None]) - elif num_colons == 1: - column_pairs.append(column.split(':')) - else: - raise ValueError('Column contains the : separator more than once') - - return column_pairs - - -class Batch(object): - """Batch class for accumulating mutations. - - :type table: :class:`Table ` - :param table: The table where mutations will be applied. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the epoch) - that all mutations will be applied at. - - :type batch_size: int - :param batch_size: (Optional) The maximum number of mutations to allow - to accumulate before committing them. - - :type transaction: bool - :param transaction: Flag indicating if the mutations should be sent - transactionally or not. If ``transaction=True`` and - an error occurs while a :class:`Batch` is active, - then none of the accumulated mutations will be - committed. If ``batch_size`` is set, the mutation - can't be transactional. - - :type wal: object - :param wal: Unused parameter (Boolean for using the HBase Write Ahead Log). - Provided for compatibility with HappyBase, but irrelevant for - Cloud Bigtable since it does not have a Write Ahead Log. - - :raises: :class:`TypeError ` if ``batch_size`` - is set and ``transaction=True``. - :class:`ValueError ` if ``batch_size`` - is not positive. - :class:`ValueError ` if ``wal`` - is used. - """ - - def __init__(self, table, timestamp=None, batch_size=None, - transaction=False, wal=_WAL_SENTINEL): - if wal is not _WAL_SENTINEL: - raise ValueError('The wal argument cannot be used with ' - 'Cloud Bigtable.') - - if batch_size is not None: - if transaction: - raise TypeError('When batch_size is set, a Batch cannot be ' - 'transactional') - if batch_size <= 0: - raise ValueError('batch_size must be positive') - - self._table = table - self._batch_size = batch_size - # Timestamp is in milliseconds, convert to microseconds. - self._timestamp = self._delete_range = None - if timestamp is not None: - self._timestamp = _microseconds_to_timestamp(1000 * timestamp) - # For deletes, we get the very next timestamp (assuming timestamp - # granularity is milliseconds). This is because HappyBase users - # expect HBase deletes to go **up to** and **including** the - # timestamp while Cloud Bigtable Time Ranges **exclude** the - # final timestamp. - next_timestamp = self._timestamp + _ONE_MILLISECOND - self._delete_range = TimestampRange(end=next_timestamp) - self._transaction = transaction - - # Internal state for tracking mutations. - self._row_map = {} - self._mutation_count = 0 - - def send(self): - """Send / commit the batch of mutations to the server.""" - for row in self._row_map.values(): - # commit() does nothing if row hasn't accumulated any mutations. - row.commit() - - self._row_map.clear() - self._mutation_count = 0 - - def _try_send(self): - """Send / commit the batch if mutations have exceeded batch size.""" - if self._batch_size and self._mutation_count >= self._batch_size: - self.send() - - def _get_row(self, row_key): - """Gets a row that will hold mutations. - - If the row is not already cached on the current batch, a new row will - be created. - - :type row_key: str - :param row_key: The row key for a row stored in the map. - - :rtype: :class:`Row ` - :returns: The newly created or stored row that will hold mutations. - """ - if row_key not in self._row_map: - table = self._table._low_level_table - self._row_map[row_key] = table.row(row_key) - - return self._row_map[row_key] - - def put(self, row, data, wal=_WAL_SENTINEL): - """Insert data into a row in the table owned by this batch. - - :type row: str - :param row: The row key where the mutation will be "put". - - :type data: dict - :param data: Dictionary containing the data to be inserted. The keys - are columns names (of the form ``fam:col``) and the values - are strings (bytes) to be stored in those columns. - - :type wal: object - :param wal: Unused parameter (to over-ride the default on the - instance). Provided for compatibility with HappyBase, but - irrelevant for Cloud Bigtable since it does not have a - Write Ahead Log. - - :raises: :class:`ValueError ` if ``wal`` - is used. - """ - if wal is not _WAL_SENTINEL: - raise ValueError('The wal argument cannot be used with ' - 'Cloud Bigtable.') - - row_object = self._get_row(row) - # Make sure all the keys are valid before beginning - # to add mutations. - column_pairs = _get_column_pairs(six.iterkeys(data), - require_qualifier=True) - for column_family_id, column_qualifier in column_pairs: - value = data[column_family_id + ':' + column_qualifier] - row_object.set_cell(column_family_id, column_qualifier, - value, timestamp=self._timestamp) - - self._mutation_count += len(data) - self._try_send() - - def _delete_columns(self, columns, row_object): - """Adds delete mutations for a list of columns and column families. - - :type columns: list - :param columns: Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type row_object: :class:`Row ` - :param row_object: The row which will hold the delete mutations. - - :raises: :class:`ValueError ` if the delete - timestamp range is set on the current batch, but a - column family delete is attempted. - """ - column_pairs = _get_column_pairs(columns) - for column_family_id, column_qualifier in column_pairs: - if column_qualifier is None: - if self._delete_range is not None: - raise ValueError('The Cloud Bigtable API does not support ' - 'adding a timestamp to ' - '"DeleteFromFamily" ') - row_object.delete_cells(column_family_id, - columns=row_object.ALL_COLUMNS) - else: - row_object.delete_cell(column_family_id, - column_qualifier, - time_range=self._delete_range) - - def delete(self, row, columns=None, wal=_WAL_SENTINEL): - """Delete data from a row in the table owned by this batch. - - :type row: str - :param row: The row key where the delete will occur. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - If not used, will delete the entire row. - - :type wal: object - :param wal: Unused parameter (to over-ride the default on the - instance). Provided for compatibility with HappyBase, but - irrelevant for Cloud Bigtable since it does not have a - Write Ahead Log. - - :raises: :class:`ValueError ` if ``wal`` - is used, or if if the delete timestamp range is set on the - current batch, but a full row delete is attempted. - """ - if wal is not _WAL_SENTINEL: - raise ValueError('The wal argument cannot be used with ' - 'Cloud Bigtable.') - - row_object = self._get_row(row) - - if columns is None: - # Delete entire row. - if self._delete_range is not None: - raise ValueError('The Cloud Bigtable API does not support ' - 'adding a timestamp to "DeleteFromRow" ' - 'mutations') - row_object.delete() - self._mutation_count += 1 - else: - self._delete_columns(columns, row_object) - self._mutation_count += len(columns) - - self._try_send() - - def __enter__(self): - """Enter context manager, no set-up required.""" - return self - - def __exit__(self, exc_type, exc_value, traceback): - """Exit context manager, no set-up required. - - :type exc_type: type - :param exc_type: The type of the exception if one occurred while the - context manager was active. Otherwise, :data:`None`. - - :type exc_value: :class:`Exception ` - :param exc_value: An instance of ``exc_type`` if an exception occurred - while the context was active. - Otherwise, :data:`None`. - - :type traceback: ``traceback`` type - :param traceback: The traceback where the exception occurred (if one - did occur). Otherwise, :data:`None`. - """ - # If the context manager encountered an exception and the batch is - # transactional, we don't commit the mutations. - if self._transaction and exc_type is not None: - return - - # NOTE: For non-transactional batches, this will even commit mutations - # if an error occurred during the context manager. - self.send() diff --git a/gcloud_bigtable/happybase/connection.py b/gcloud_bigtable/happybase/connection.py deleted file mode 100644 index 7876f05..0000000 --- a/gcloud_bigtable/happybase/connection.py +++ /dev/null @@ -1,467 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase connection module.""" - - -import datetime -import six - -from gcloud_bigtable.client import Client -from gcloud_bigtable.column_family import GarbageCollectionRule -from gcloud_bigtable.column_family import GarbageCollectionRuleIntersection -from gcloud_bigtable.happybase.table import Table -from gcloud_bigtable.table import Table as _LowLevelTable - - -# Constants reproduced here for compatibility, though values are -# all null. -COMPAT_MODES = None -THRIFT_TRANSPORTS = None -THRIFT_PROTOCOLS = None -DEFAULT_HOST = None -DEFAULT_PORT = None -DEFAULT_TRANSPORT = None -DEFAULT_COMPAT = None -DEFAULT_PROTOCOL = None - - -def _get_cluster(timeout=None): - """Gets cluster for the default project. - - Creates a client with the inferred credentials and project ID from - the local environment. Then uses :meth:`.Client.list_clusters` to - get the unique cluster owned by the project. - - If the request fails for any reason, or if there isn't exactly one cluster - owned by the project, then this function will fail. - - :type timeout: int - :param timeout: (Optional) The socket timeout in milliseconds. - - :rtype: :class:`gcloud_bigtable.cluster.Cluster` - :returns: The unique cluster owned by the project inferred from - the environment. - :raises: :class:`ValueError ` if any of the unused - """ - client_kwargs = {'admin': True} - if timeout is not None: - client_kwargs['timeout_seconds'] = timeout / 1000.0 - client = Client(**client_kwargs) - client.start() - clusters, failed_zones = client.list_clusters() - client.stop() - - if len(failed_zones) != 0: - raise ValueError('Determining cluster via ListClusters encountered ' - 'failed zones.') - if len(clusters) == 0: - raise ValueError('This client doesn\'t have access to any clusters.') - if len(clusters) > 1: - raise ValueError('This client has access to more than one cluster. ' - 'Please directly pass the cluster you\'d ' - 'like to use.') - return clusters[0] - - -def _parse_family_option(option): - """Parses a column family option into a garbage collection rule. - - .. note:: - - If ``option`` is not a dictionary, the type is not checked. - If ``option`` is :data:`None`, there is nothing to do, since this - is the correct output. - - :type option: :class:`dict`, - :data:`NoneType `, - :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleUnion`, - :class:`.GarbageCollectionRuleIntersection` - :param option: A column family option passes as a dictionary value in - :meth:`Connection.create_table`. - - :rtype: :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleUnion`, - :class:`.GarbageCollectionRuleIntersection` - :returns: A garbage collection rule parsed from the input. - :raises: :class:`ValueError ` if ``option`` is a - dictionary but keys other than ``max_versions`` and - ``time_to_live`` are used. - """ - result = option - if isinstance(result, dict): - if not set(result.keys()) <= set(['max_versions', 'time_to_live']): - raise ValueError('Cloud Bigtable only supports max_versions and ' - 'time_to_live column family settings', - 'Received', result.keys()) - - max_num_versions = result.get('max_versions') - max_age = None - if 'time_to_live' in result: - max_age = datetime.timedelta(seconds=result['time_to_live']) - - if len(result) == 0: - result = None - elif len(result) == 1: - if max_num_versions is None: - result = GarbageCollectionRule(max_age=max_age) - else: - result = GarbageCollectionRule( - max_num_versions=max_num_versions) - else: # By our check above we know this means len(result) == 2. - rule1 = GarbageCollectionRule(max_age=max_age) - rule2 = GarbageCollectionRule(max_num_versions=max_num_versions) - result = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) - - return result - - -class Connection(object): - """Connection to Cloud Bigtable backend. - - .. note:: - - If you pass a ``cluster``, it will be :meth:`.Cluster.copy`-ed before - being stored on the new connection. This also copies the - :class:`.Client` that created the :class:`.Cluster` instance and the - :class:`Credentials ` stored on the - client. - - :type host: :data:`NoneType ` - :param host: Unused parameter. Provided for compatibility with HappyBase, - but irrelevant for Cloud Bigtable since it has a fixed host. - - :type port: :data:`NoneType ` - :param port: Unused parameter. Provided for compatibility with HappyBase, - but irrelevant for Cloud Bigtable since it has a fixed host. - - :type timeout: int - :param timeout: (Optional) The socket timeout in milliseconds. - - :type autoconnect: bool - :param autoconnect: Whether the connection should be :meth:`open`-ed - during construction. - - :type table_prefix: str - :param table_prefix: (Optional) Prefix used to construct table names. - - :type table_prefix_separator: str - :param table_prefix_separator: Separator used with ``table_prefix``. - - :type compat: :data:`NoneType ` - :param compat: Unused parameter. Provided for compatibility with - HappyBase, but irrelevant for Cloud Bigtable since there - is only one version. - - :type transport: :data:`NoneType ` - :param transport: Unused parameter. Provided for compatibility with - HappyBase, but irrelevant for Cloud Bigtable since the - transport is fixed. - - :type protocol: :data:`NoneType ` - :param protocol: Unused parameter. Provided for compatibility with - HappyBase, but irrelevant for Cloud Bigtable since the - protocol is fixed. - - :type cluster: :class:`gcloud_bigtable.cluster.Cluster` - :param cluster: (Optional) A Cloud Bigtable cluster. The instance also - owns a client for making gRPC requests to the Cloud - Bigtable API. If not passed in, defaults to creating client - with ``admin=True`` and using the ``timeout`` for the - ``timeout_seconds``. The credentials for the client - will be the implicit ones loaded from the environment. - Then that client is used to retrieve all the clusters - owned by the client's project. - - :raises: :class:`ValueError ` if any of the unused - parameters are specified with a value other than the defaults. - :class:`TypeError ` if ``table_prefix`` or - ``table_prefix_separator`` are provided but not strings. - """ - - def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, timeout=None, - autoconnect=True, table_prefix=None, - table_prefix_separator='_', compat=DEFAULT_COMPAT, - transport=DEFAULT_TRANSPORT, protocol=DEFAULT_PROTOCOL, - cluster=None): - if host is not DEFAULT_HOST: - raise ValueError('Host cannot be set for gcloud HappyBase module') - if port is not DEFAULT_PORT: - raise ValueError('Port cannot be set for gcloud HappyBase module') - if compat is not DEFAULT_COMPAT: - raise ValueError('Compat cannot be set for gcloud ' - 'HappyBase module') - if transport is not DEFAULT_TRANSPORT: - raise ValueError('Transport cannot be set for gcloud ' - 'HappyBase module') - if protocol is not DEFAULT_PROTOCOL: - raise ValueError('Protocol cannot be set for gcloud ' - 'HappyBase module') - - if table_prefix is not None: - if not isinstance(table_prefix, six.string_types): - raise TypeError('table_prefix must be a string', 'received', - table_prefix, type(table_prefix)) - - if not isinstance(table_prefix_separator, six.string_types): - raise TypeError('table_prefix_separator must be a string', - 'received', table_prefix_separator, - type(table_prefix_separator)) - - self.table_prefix = table_prefix - self.table_prefix_separator = table_prefix_separator - - if cluster is None: - self._cluster = _get_cluster(timeout=timeout) - else: - if timeout is not None: - raise ValueError('Timeout cannot be used when an existing ' - 'cluster is passed') - self._cluster = cluster.copy() - - if autoconnect: - self.open() - - self._initialized = True - - def open(self): - """Open the underlying transport to Cloud Bigtable. - - This method opens the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by - this connection. - """ - self._cluster.client.start() - - def close(self): - """Close the underlying transport to Cloud Bigtable. - - This method closes the underlying HTTP/2 gRPC connection using a - :class:`.Client` bound to the :class:`.Cluster` owned by - this connection. - """ - self._cluster.client.stop() - - def __del__(self): - try: - self._initialized - except AttributeError: - # Failure from constructor - return - else: - self.close() - - def _table_name(self, name): - """Construct a table name by optionally adding a table name prefix. - - :type name: str - :param name: The name to have a prefix added to it. - - :rtype: str - :returns: The prefixed name, if the current connection has a table - prefix set. - """ - if self.table_prefix is None: - return name - - return self.table_prefix + self.table_prefix_separator + name - - def table(self, name, use_prefix=True): - """Table factory. - - :type name: str - :param name: The name of the table to be created. - - :type use_prefix: bool - :param use_prefix: Whether to use the table prefix (if any). - - :rtype: `Table ` - :returns: Table instance owned by this connection. - """ - if use_prefix: - name = self._table_name(name) - return Table(name, self) - - def tables(self): - """Return a list of table names available to this connection. - - .. note:: - - This lists every table in the cluster owned by this connection, - **not** every table that a given user may have access to. - - .. note:: - - If ``table_prefix`` is set on this connection, only returns the - table names which match that prefix. - - :rtype: list - :returns: List of string table names. - """ - low_level_table_instances = self._cluster.list_tables() - table_names = [table_instance.table_id - for table_instance in low_level_table_instances] - - # Filter using prefix, and strip prefix from names - if self.table_prefix is not None: - prefix = self._table_name('') - offset = len(prefix) - table_names = [name[offset:] for name in table_names - if name.startswith(prefix)] - - return table_names - - def create_table(self, name, families): - """Create a table. - - .. warning:: - - The only column family options from HappyBase that are able to be - used with Cloud Bigtable are ``max_versions`` and ``time_to_live``. - - .. note:: - - This method is **not** atomic. The Cloud Bigtable API separates - the creation of a table from the creation of column families. Thus - this method needs to send 1 request for the table creation and 1 - request for each column family. If any of these fails, the method - will fail, but the progress made towards completion cannot be - rolled back. - - Values in ``families`` represent column family options. In HappyBase, - these are dictionaries, corresponding to the ``ColumnDescriptor`` - structure in the Thrift API. The accepted keys are: - - * ``max_versions`` (``int``) - * ``compression`` (``str``) - * ``in_memory`` (``bool``) - * ``bloom_filter_type`` (``str``) - * ``bloom_filter_vector_size`` (``int``) - * ``bloom_filter_nb_hashes`` (``int``) - * ``block_cache_enabled`` (``bool``) - * ``time_to_live`` (``int``) - - :type name: str - :param name: The name of the table to be created. - - :type families: dict - :param families: Dictionary with column family names as keys and column - family options as the values. The options can be among - - * :class:`dict` - * :class:`.GarbageCollectionRule` - * :class:`.GarbageCollectionRuleUnion` - * :class:`.GarbageCollectionRuleIntersection` - - :raises: :class:`TypeError ` if ``families`` is - not a dictionary, - :class:`ValueError ` if ``families`` - has no entries - """ - if not isinstance(families, dict): - raise TypeError('families arg must be a dictionary') - - if not families: - raise ValueError('Cannot create table %r (no column ' - 'families specified)' % (name,)) - - # Parse all keys before making any API requests. - gc_rule_dict = {} - for column_family_name, option in families.items(): - if column_family_name.endswith(':'): - column_family_name = column_family_name[:-1] - gc_rule_dict[column_family_name] = _parse_family_option(option) - - # Create table instance and then make API calls. - name = self._table_name(name) - low_level_table = _LowLevelTable(name, self._cluster) - low_level_table.create() - - for column_family_name, gc_rule in gc_rule_dict.items(): - column_family = low_level_table.column_family( - column_family_name, gc_rule=gc_rule) - column_family.create() - - def delete_table(self, name, disable=False): - """Delete the specified table. - - :type name: str - :param name: The name of the table to be deleted. If ``table_prefix`` - is set, a prefix will be added to the ``name``. - - :type disable: bool - :param disable: Whether to first disable the table if needed. This - is provided for compatibility with HappyBase, but is - not relevant for Cloud Bigtable since it has no concept - of enabled / disabled tables. - - :raises: :class:`ValueError ` - if ``disable=True``. - """ - if disable: - raise ValueError('The disable argument should not be used in ' - 'delete_table(). Cloud Bigtable has no concept ' - 'of enabled / disabled tables.') - - name = self._table_name(name) - _LowLevelTable(name, self._cluster).delete() - - def enable_table(self, name): - """Enable the specified table. - - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API has no concept of ' - 'enabled or disabled tables.') - - def disable_table(self, name): - """Disable the specified table. - - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API has no concept of ' - 'enabled or disabled tables.') - - def is_table_enabled(self, name): - """Return whether the specified table is enabled. - - Cloud Bigtable has no concept of enabled / disabled tables so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API has no concept of ' - 'enabled or disabled tables.') - - def compact_table(self, name, major=False): - """Compact the specified table. - - Cloud Bigtable does not support compacting a table, so this - method does not work. It is provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API does not support ' - 'compacting a table.') diff --git a/gcloud_bigtable/happybase/notes.txt b/gcloud_bigtable/happybase/notes.txt deleted file mode 100644 index 183e144..0000000 --- a/gcloud_bigtable/happybase/notes.txt +++ /dev/null @@ -1,82 +0,0 @@ -# Installing HBase / Thrift locally - - 1. Run `javac -version` to make sure you have JDK 7 (or 8) - For example "javac 1.7.0_79" == JDK 7. - 2. Visit http://apache.cs.utah.edu/hbase/ (or other mirror) - and find your latest version (e.g. - http://apache.cs.utah.edu/hbase/1.1.1/hbase-1.1.1-bin.tar.gz) - Make sure it is "-bin" and not "-src". - 3. Run `wget http://apache.cs.utah.edu/hbase/1.1.1/hbase-1.1.1-bin.tar.gz` - (HOLY SHIT I CANNOT BELIEVE NONE OF THE MIRRORS ARE HTTPS) - 4. Run `tar -zxvf hbase-1.1.1-bin.tar.gz` (maybe drop the -v flag) - 5. Set the JAVA_HOME environment variable (e.g. if `which java` tells - us it is in `/usr/bin/java` then `export JAVA_HOME=/usr` - 6. Edit `hbase-1.1.1/conf/hbase-site.xml` to give locations on the local - filesystem where HBase and ZooKeeper are allowed to write data - - - - hbase.rootdir - file://.../hbase_throwaway/hbase - - - hbase.zookeeper.property.dataDir - .../hbase_throwaway/zookeeper - - - - **DO NOT** create the HBase data directory. HBase will do this for you. - If you create the directory, HBase will attempt to do a migration, which - is not what you want. - - 7. Run `./hbase-1.1.1/bin/start-hbase.sh`. You can use the `jps` command to - verify that you have one running process called `HMaster`. (`jps` == - Java Process Status tool) - 8. Feel free to play around in the shell `./hbase-1.1.1/bin/hbase shell` - 9. Run `./hbase-1.1.1/bin/stop-hbase.sh` to step the server -10. Run `./hbase-1.1.1/bin/hbase thrift start` - (H/T: http://wiki.apache.org/hadoop/Hbase/ThriftApi) - -# Installing HappyBase locally - - 1. Run `virtualenv happybase-venv` - 2. Run `source happybase-venv/bin/activate` - 3. Run `pip install happybase` - 4. Run `python -c "import happybase"` to make sure it worked - 5. Run `pip install ipython` if you prefer that as your shell - in your virtualenv - -# Running HappyBase locally - - 1. Run `./hbase-1.1.1/bin/start-hbase.sh` - 2. Run `./hbase-1.1.1/bin/hbase thrift start --port=9090 > ./hbase-1.1.1/logs/thrift.log 2>&1 &` - (in the background, use `jps` to get the process ID) - 3. Run Python: - - import happybase - connection = happybase.Connection(host='localhost', port=9090) - print(connection.tables()) - families = {'family': {}} - table_name = 'table-name' - connection.create_table(table_name, families) - print(connection.tables()) - table = connection.table(table_name) - - row_key = 'row-key' - table.put(row_key, {'family:qual1': 'value1', - 'family:qual2': 'value2'}) - row = table.row(row_key) - print(type(row)) - - for key, data in table.scan(row_prefix='row'): - print((key, data)) - - table.put(row_key, {'family:qual1': 'value1-new', - 'family:qual2': 'value2-new'}) - row = table.row(row_key, include_timestamp=True) - print(row) - print(table.cells(row_key, 'family:qual1', include_timestamp=True)) - print(table.cells(row_key, 'family:qual2', include_timestamp=True)) - - print(list(table.scan())) - print(list(table.scan(sorted_columns=True))) diff --git a/gcloud_bigtable/happybase/pool.py b/gcloud_bigtable/happybase/pool.py deleted file mode 100644 index 330c5de..0000000 --- a/gcloud_bigtable/happybase/pool.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase pool module.""" - - -import contextlib -import six -import threading - -from gcloud_bigtable.happybase.connection import Connection -from gcloud_bigtable.happybase.connection import _get_cluster - - -_MIN_POOL_SIZE = 1 -"""Minimum allowable size of a connection pool.""" - - -class NoConnectionsAvailable(RuntimeError): - """Exception raised when no connections are available. - - This happens if a timeout was specified when obtaining a connection, - and no connection became available within the specified timeout. - """ - - -class ConnectionPool(object): - """Thread-safe connection pool. - - .. note:: - - All keyword arguments are passed unmodified to the - :class:`.Connection` constructor **except** for ``autoconnect``. - This is because the ``open`` / ``closed`` status of a connection - is managed by the pool. In addition, if ``cluster`` is not passed, - the default / inferred cluster is determined by the pool and then - passed to each :class:`.Connection` that is created. - - :type size: int - :param size: The maximum number of concurrently open connections. - - :type kwargs: dict - :param kwargs: Keyword arguments passed to :class:`.Connection` - constructor. - - :raises: :class:`TypeError ` if ``size`` - is non an integer. - :class:`ValueError ` if ``size`` - is not positive. - """ - def __init__(self, size, **kwargs): - if not isinstance(size, six.integer_types): - raise TypeError('Pool size arg must be an integer') - - if size < _MIN_POOL_SIZE: - raise ValueError('Pool size must be positive') - - self._lock = threading.Lock() - self._queue = six.moves.queue.LifoQueue(maxsize=size) - self._thread_connections = threading.local() - - connection_kwargs = kwargs - connection_kwargs['autoconnect'] = False - if 'cluster' not in connection_kwargs: - connection_kwargs['cluster'] = _get_cluster( - timeout=kwargs.get('timeout')) - - for _ in six.moves.range(size): - connection = Connection(**connection_kwargs) - self._queue.put(connection) - - def _acquire_connection(self, timeout=None): - """Acquire a connection from the pool. - - :type timeout: int - :param timeout: (Optional) Time (in seconds) to wait for a connection - to open. - - :rtype: :class:`.Connection` - :returns: An active connection from the queue stored on the pool. - :raises: :class:`NoConnectionsAvailable` if ``Queue.get`` fails - before the ``timeout`` (only if a timeout is specified). - """ - try: - return self._queue.get(block=True, timeout=timeout) - except six.moves.queue.Empty: - raise NoConnectionsAvailable('No connection available from pool ' - 'within specified timeout') - - @contextlib.contextmanager - def connection(self, timeout=None): - """Obtain a connection from the pool. - - Must be used as a context manager, for example:: - - with pool.connection() as connection: - pass # do something with the connection - - If ``timeout`` is omitted, this method waits forever for a connection - to become available. - - :type timeout: int - :param timeout: (Optional) Time (in seconds) to wait for a connection - to open. - - :rtype: :class:`.Connection` - :returns: An active connection from the pool. - :raises: :class:`NoConnectionsAvailable` if no connection can be - retrieved from the pool before the ``timeout`` (only if - a timeout is specified). - """ - connection = getattr(self._thread_connections, 'current', None) - - retrieved_new_cnxn = False - if connection is None: - # In this case we need to actually grab a connection from the - # pool. After retrieval, the connection is stored on a thread - # local so that nested connection requests from the same - # thread can re-use the same connection instance. - # - # NOTE: This code acquires a lock before assigning to the - # thread local; see - # ('https://emptysqua.re/blog/' - # 'another-thing-about-pythons-threadlocals/') - retrieved_new_cnxn = True - connection = self._acquire_connection(timeout) - with self._lock: - self._thread_connections.current = connection - - # This is a no-op for connections that have already been opened - # since they just call Client.start(). - connection.open() - yield connection - - # Remove thread local reference after the outermost 'with' block - # ends. Afterwards the thread no longer owns the connection. - if retrieved_new_cnxn: - del self._thread_connections.current - self._queue.put(connection) diff --git a/gcloud_bigtable/happybase/table.py b/gcloud_bigtable/happybase/table.py deleted file mode 100644 index c496740..0000000 --- a/gcloud_bigtable/happybase/table.py +++ /dev/null @@ -1,921 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Google Cloud Bigtable HappyBase table module.""" - - -import six -import struct - -from gcloud_bigtable._helpers import _microseconds_to_timestamp -from gcloud_bigtable._helpers import _timestamp_to_microseconds -from gcloud_bigtable._helpers import _to_bytes -from gcloud_bigtable.column_family import GarbageCollectionRule -from gcloud_bigtable.column_family import GarbageCollectionRuleIntersection -from gcloud_bigtable.happybase.batch import Batch -from gcloud_bigtable.happybase.batch import _WAL_SENTINEL -from gcloud_bigtable.happybase.batch import _get_column_pairs -from gcloud_bigtable.row import RowFilter -from gcloud_bigtable.row import RowFilterChain -from gcloud_bigtable.row import RowFilterUnion -from gcloud_bigtable.row import TimestampRange -from gcloud_bigtable.table import Table as _LowLevelTable - - -_UNPACK_I64 = struct.Struct('>q').unpack -_DEFAULT_BATCH_SIZE = object() -_DEFAULT_SCAN_BATCHING = object() -_DEFAULT_SORTED_COLUMNS = object() - - -def make_row(cell_map, include_timestamp): - """Make a row dict for a Thrift cell mapping. - - .. note:: - - This method is only provided for HappyBase compatibility, but does not - actually work. - - :type cell_map: dict - :param cell_map: Dictionary with ``fam:col`` strings as keys and ``TCell`` - instances as values. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API output is not the same ' - 'as the output from the Thrift server, so this ' - 'helper can not be implemented.', 'Called with', - cell_map, include_timestamp) - - -def make_ordered_row(sorted_columns, include_timestamp): - """Make a row dict for sorted Thrift column results from scans. - - .. note:: - - This method is only provided for HappyBase compatibility, but does not - actually work. - - :type sorted_columns: list - :param sorted_columns: List of ``TColumn`` instances from Thrift. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API output is not the same ' - 'as the output from the Thrift server, so this ' - 'helper can not be implemented.', 'Called with', - sorted_columns, include_timestamp) - - -def _gc_rule_to_dict(gc_rule): - """Converts garbage collection rule to dictionary if possible. - - This is in place to support dictionary values was was done - in HappyBase, which has somewhat different garbage collection rule - settings for column families. - - Only does this if the garbage collection rule is: - - * Simple :class:`.GarbageCollectionRule` with ``max_age`` - * Simple :class:`.GarbageCollectionRule` with ``max_num_versions`` - * Composite :class:`.GarbageCollectionRuleIntersection` with - two rules each for ``max_age`` and ``max_num_versions`` - - Otherwise, just returns the input without change. - - :type gc_rule: :data:`NoneType `, - :class:`.GarbageCollectionRule`, - :class:`.GarbageCollectionRuleIntersection`, or - :class:`.GarbageCollectionRuleUnion` - :param gc_rule: A garbae collection rule to convert to a dictionary - (if possible). - - :rtype: dict, - :class:`.GarbageCollectionRuleIntersection`, or - :class:`.GarbageCollectionRuleUnion` - :returns: The converted garbage collection rule. - """ - result = gc_rule - if gc_rule is None: - result = {} - elif isinstance(gc_rule, GarbageCollectionRule): - result = {} - # We assume that the GC rule has a single value. - if gc_rule.max_num_versions is not None: - result['max_versions'] = gc_rule.max_num_versions - if gc_rule.max_age is not None: - result['time_to_live'] = gc_rule.max_age.total_seconds() - elif isinstance(gc_rule, GarbageCollectionRuleIntersection): - if len(gc_rule.rules) == 2: - rule1, rule2 = gc_rule.rules - if (isinstance(rule1, GarbageCollectionRule) and - isinstance(rule2, GarbageCollectionRule)): - rule1 = _gc_rule_to_dict(rule1) - rule2 = _gc_rule_to_dict(rule2) - key1, = rule1.keys() - key2, = rule2.keys() - if key1 != key2: - result = {key1: rule1[key1], key2: rule2[key2]} - return result - - -def _convert_to_time_range(timestamp=None): - """Create a timestamp range from an HBase / HappyBase timestamp. - - HBase uses timestamp as an argument to specify an exclusive end - deadline. Cloud Bigtable also uses exclusive end times, so - the behavior matches. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). Intended to be used as the end of an HBase - time range, which is exclusive. - - :rtype: :class:`.TimestampRange`, :data:`NoneType ` - :returns: The timestamp range corresponding to the passed in - ``timestamp``. - """ - if timestamp is None: - return None - - next_timestamp = _microseconds_to_timestamp(1000 * timestamp) - return TimestampRange(end=next_timestamp) - - -def _cells_to_pairs(cells, include_timestamp=False): - """Converts list of cells to HappyBase format. - - :type cells: list - :param cells: List of :class:`.Cell` returned from a read request. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: list - :returns: List of values in the cell. If ``include_timestamp=True``, each - value will be a pair, with the first part the bytes value in - the cell and the second part the number of milliseconds in the - timestamp on the cell. - """ - result = [] - for cell in cells: - if include_timestamp: - ts_millis = _timestamp_to_microseconds(cell.timestamp) // 1000 - result.append((cell.value, ts_millis)) - else: - result.append(cell.value) - return result - - -def _filter_chain_helper(column=None, versions=None, timestamp=None, - filters=None): - """Create filter chain to limit a results set. - - :type column: str - :param column: (Optional) The column (``fam:col``) to be selected - with the filter. - - :type versions: int - :param versions: (Optional) The maximum number of cells to return. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be matched. - - :type filters: list - :param filters: (Optional) List of existing filters to be extended. - - :rtype: :class:`.RowFilterChain`, :class:`.RowFilter` - :returns: The chained filter created, or just a single filter if only - one was needed. - :raises: :class:`ValueError ` if there are no - filters to chain. - """ - if filters is None: - filters = [] - - if column is not None: - column_family_id, column_qualifier = column.split(':') - fam_filter = RowFilter(family_name_regex_filter=column_family_id) - qual_filter = RowFilter(column_qualifier_regex_filter=column_qualifier) - filters.extend([fam_filter, qual_filter]) - if versions is not None: - filters.append(RowFilter(cells_per_column_limit_filter=versions)) - time_range = _convert_to_time_range(timestamp=timestamp) - if time_range is not None: - filters.append(RowFilter(timestamp_range_filter=time_range)) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterChain(filters=filters) - - -def _columns_filter_helper(columns): - """Creates a union filter for a list of columns. - - :type columns: list - :param columns: Iterable containing column names (as strings). Each column - name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :rtype: :class:`.RowFilterUnion`, :class:`.RowFilter` - :returns: The union filter created containing all of the matched columns. - :raises: :class:`ValueError ` if there are no - filters to union. - """ - filters = [] - for column_family_id, column_qualifier in _get_column_pairs(columns): - if column_qualifier is not None: - fam_filter = RowFilter(family_name_regex_filter=column_family_id) - qual_filter = RowFilter( - column_qualifier_regex_filter=column_qualifier) - combined_filter = RowFilterChain( - filters=[fam_filter, qual_filter]) - filters.append(combined_filter) - else: - fam_filter = RowFilter(family_name_regex_filter=column_family_id) - filters.append(fam_filter) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterUnion(filters=filters) - - -def _row_keys_filter_helper(row_keys): - """Creates a union filter for a list of rows. - - :type row_keys: list - :param row_keys: Iterable containing row keys (as strings). - - :rtype: :class:`.RowFilterUnion`, :class:`.RowFilter` - :returns: The union filter created containing all of the row keys. - :raises: :class:`ValueError ` if there are no - filters to union. - """ - filters = [] - for row_key in row_keys: - filters.append(RowFilter(row_key_regex_filter=row_key)) - - num_filters = len(filters) - if num_filters == 0: - raise ValueError('Must have at least one filter.') - elif num_filters == 1: - return filters[0] - else: - return RowFilterUnion(filters=filters) - - -def _partial_row_to_dict(partial_row_data, include_timestamp=False): - """Convert a low-level row data object to a dictionary. - - Assumes only the latest value in each row are needed (e.g. different - behavior than in :meth:`Table.cells`). - - :type partial_row_data: :class:`.row_data.PartialRowData` - :param partial_row_data: Row data consumed from a stream. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: dict - :returns: The row data converted to a dictionary. - """ - result = {} - for column, cells in six.iteritems(partial_row_data.to_dict()): - cell_vals = _cells_to_pairs(cells, - include_timestamp=include_timestamp) - # NOTE: We assume there is exactly 1 version since we used that in - # our filter, but we don't check this. - result[column] = cell_vals[0] - return result - - -def _next_char(str_val, index): - """Gets the next character based on a position in a string. - - :type str_val: str - :param str_val: A string containing the character to update. - - :type index: int - :param index: An integer index in ``str_val``. - - :rtype: str - :returns: The next character after the character at ``index`` - in ``str_val``. - """ - if six.PY3: # pragma: NO COVER - ord_val = str_val[index] - else: - ord_val = ord(str_val[index]) - return _to_bytes(chr(ord_val + 1), encoding='latin-1') - - -def _string_successor(str_val): - """Increment and truncate a byte string. - - Determines shortest string that sorts after the given string when - compared using regular string comparison semantics. - - Borrowed from gcloud-golang. - - Increments the last byte that is smaller than ``0xFF``, and - drops everything after it. If the string only contains ``0xFF`` bytes, - ``''`` is returned. - - :type str_val: str - :param str_val: String to increment. - - :rtype: str - :returns: The next string in lexical order after ``str_val``. - """ - str_val = _to_bytes(str_val, encoding='latin-1') - if str_val == b'': - return str_val - - index = len(str_val) - 1 - while index >= 0: - if six.PY3: # pragma: NO COVER - if str_val[index] != 0xff: - break - else: - if str_val[index] != b'\xff': - break - index -= 1 - - if index == -1: - return b'' - - return str_val[:index] + _next_char(str_val, index) - - -class Table(object): - """Representation of Cloud Bigtable table. - - Used for adding data and - - :type name: str - :param name: The name of the table. - - :type connection: :class:`.Connection` - :param connection: The connection which has access to the table. - """ - - def __init__(self, name, connection): - self.name = name - # This remains as legacy for HappyBase, but only the cluster - # from it is needed. - self.connection = connection - self._low_level_table = None - if self.connection is not None: - self._low_level_table = _LowLevelTable(self.name, - self.connection._cluster) - - def __repr__(self): - return '' % (self.name,) - - def families(self): - """Retrieve the column families for this table. - - :rtype: dict - :returns: Mapping from column family name to garbage collection rule - for a column family. - """ - column_family_map = self._low_level_table.list_column_families() - return {col_fam: _gc_rule_to_dict(col_fam_obj.gc_rule) - for col_fam, col_fam_obj in column_family_map.items()} - - def regions(self): - """Retrieve the regions for this table. - - Cloud Bigtable does not give information about how a table is laid - out in memory, so regions so this method does not work. It is - provided simply for compatibility. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('The Cloud Bigtable API does not have a ' - 'concept of splitting a table into regions.') - - def row(self, row, columns=None, timestamp=None, include_timestamp=False): - """Retrieve a single row of data. - - Returns the latest cells in each column (or all columns if ``columns`` - is not specified). If a ``timestamp`` is set, then **latest** becomes - **latest** up until ``timestamp``. - - :type row: str - :param row: Row key for the row we are reading from. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before the - the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: dict - :returns: Dictionary containing all the latest column values in - the row. - """ - filters = [] - if columns is not None: - filters.append(_columns_filter_helper(columns)) - # versions == 1 since we only want the latest. - filter_ = _filter_chain_helper(versions=1, timestamp=timestamp, - filters=filters) - - partial_row_data = self._low_level_table.read_row( - row, filter_=filter_) - if partial_row_data is None: - return {} - - return _partial_row_to_dict(partial_row_data, - include_timestamp=include_timestamp) - - def rows(self, rows, columns=None, timestamp=None, - include_timestamp=False): - """Retrieve multiple rows of data. - - All optional arguments behave the same in this method as they do in - :meth:`row`. - - :type rows: list - :param rows: Iterable of the row keys for the rows we are reading from. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: list - :returns: A list of pairs, where the first is the row key and the - second is a dictionary with the filtered values returned. - """ - if not rows: - # Avoid round-trip if the result is empty anyway - return [] - - filters = [] - if columns is not None: - filters.append(_columns_filter_helper(columns)) - filters.append(_row_keys_filter_helper(rows)) - # versions == 1 since we only want the latest. - filter_ = _filter_chain_helper(versions=1, timestamp=timestamp, - filters=filters) - - partial_rows_data = self._low_level_table.read_rows(filter_=filter_) - # NOTE: We could use max_loops = 1000 or some similar value to ensure - # that the stream isn't open too long. - partial_rows_data.consume_all() - - result = [] - for row_key in rows: - if row_key not in partial_rows_data.rows: - continue - curr_row_data = partial_rows_data.rows[row_key] - curr_row_dict = _partial_row_to_dict( - curr_row_data, include_timestamp=include_timestamp) - result.append((row_key, curr_row_dict)) - - return result - - def cells(self, row, column, versions=None, timestamp=None, - include_timestamp=False): - """Retrieve multiple versions of a single cell from the table. - - :type row: str - :param row: Row key for the row we are reading from. - - :type column: str - :param column: Column we are reading from; of the form ``fam:col``. - - :type versions: int - :param versions: (Optional) The maximum number of cells to return. If - not set, returns all cells found. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :rtype: list - :returns: List of values in the cell (with timestamps if - ``include_timestamp`` is :data:`True`). - """ - filter_ = _filter_chain_helper(column=column, versions=versions, - timestamp=timestamp) - partial_row_data = self._low_level_table.read_row(row, filter_=filter_) - if partial_row_data is None: - return [] - else: - cells = partial_row_data._cells - # We know that `_filter_chain_helper` has already verified that - # column will split as such. - column_family_id, column_qualifier = column.split(':') - # NOTE: We expect the only key in `cells` is `column_family_id` - # and the only key `cells[column_family_id]` is - # `column_qualifier`. But we don't check that this is true. - curr_cells = cells[column_family_id][column_qualifier] - return _cells_to_pairs( - curr_cells, include_timestamp=include_timestamp) - - def scan(self, row_start=None, row_stop=None, row_prefix=None, - columns=None, filter=None, timestamp=None, - include_timestamp=False, batch_size=_DEFAULT_BATCH_SIZE, - scan_batching=_DEFAULT_SCAN_BATCHING, - limit=None, sorted_columns=_DEFAULT_SORTED_COLUMNS): - """Create a scanner for data in this table. - - This method returns a generator that can be used for looping over the - matching rows. - - If ``row_prefix`` is specified, only rows with row keys matching the - prefix will be returned. If given, ``row_start`` and ``row_stop`` - cannot be used. - - .. note:: - - Both ``row_start`` and ``row_stop`` can be :data:`None` to specify - the start and the end of the table respectively. If both are - omitted, a full table scan is done. Note that this usually results - in severe performance problems. - - :type row_start: str - :param row_start: (Optional) Row key where the scanner should start - (includes ``row_start``). If not specified, reads - from the first key. If the table does not contain - ``row_start``, it will start from the next key after - it that **is** contained in the table. - - :type row_stop: str - :param row_stop: (Optional) Row key where the scanner should stop - (excludes ``row_stop``). If not specified, reads - until the last key. The table does not have to contain - ``row_stop``. - - :type row_prefix: str - :param row_prefix: (Optional) Prefix to match row keys. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` - :param filter: (Optional) An additional filter (beyond column and - row range filters supported here). HappyBase / HBase - users will have used this as an HBase filter string. See - http://hbase.apache.org/0.94/book/thrift.html - for more details on those filters. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch). If specified, only cells returned before (or - at) the timestamp will be returned. - - :type include_timestamp: bool - :param include_timestamp: Flag to indicate if cell timestamps should be - included with the output. - - :type batch_size: int - :param batch_size: Unused parameter. This determines the number of - results to retrieve per request. The HBase scanner - defaults to reading one record at a time, so this - increases that number. The Cloud Bigtable API uses - HTTP/2 streaming and this value can't be set. - - :type scan_batching: bool - :param scan_batching: Unused parameter. Provided for compatibility - with HappyBase, but irrelevant for Cloud Bigtable - since it does not have concepts of batching or - caching for scans. - - :type limit: int - :param limit: (Optional) Maximum number of rows to return. - - :type sorted_columns: bool - :param sorted_columns: Unused parameter. Provided compatibility with - HappyBase, but irrelevant for Cloud Bigtable - since it cannot return sorted columns. - - :raises: :class:`ValueError ` if ``batch_size`` - or ``scan_batching`` are used, or if ``limit`` is set but - non-positive, or if row prefix is used with row start/stop, - :class:`NotImplementedError ` - temporarily until the method is implemented, - :class:`TypeError ` if a string - ``filter`` is used. - """ - if batch_size is not _DEFAULT_BATCH_SIZE: - raise ValueError('Batch size cannot be set for gcloud ' - 'HappyBase module') - if scan_batching is not _DEFAULT_SCAN_BATCHING: - raise ValueError('Scan batching cannot be set for gcloud ' - 'HappyBase module') - if sorted_columns is not _DEFAULT_SORTED_COLUMNS: - raise ValueError('Sorted columns cannot be set for gcloud ' - 'HappyBase module') - if limit is not None and limit < 1: - raise ValueError('limit must be positive') - if row_prefix is not None: - if row_start is not None or row_stop is not None: - raise ValueError('row_prefix cannot be combined with ' - 'row_start or row_stop') - row_start = row_prefix - row_stop = _string_successor(row_prefix) - - filters = [] - if isinstance(filter, six.string_types): - raise TypeError('HBase filter strings not supported by Cloud ' - 'Bigtable. RowFilter\'s from row module may be ' - 'used instead.') - elif filter is not None: - filters.append(filter) - - if columns is not None: - filters.append(_columns_filter_helper(columns)) - # versions == 1 since we only want the latest. - filter_ = _filter_chain_helper(versions=1, timestamp=timestamp, - filters=filters) - - partial_rows_data = self._low_level_table.read_rows( - start_key=row_start, end_key=row_stop, - limit=limit, filter_=filter_) - - # Mutable copy of data. - rows_dict = partial_rows_data.rows - while True: - try: - partial_rows_data.consume_next() - row_key, curr_row_data = rows_dict.popitem() - # NOTE: We expect len(rows_dict) == 0, but don't check it. - curr_row_dict = _partial_row_to_dict( - curr_row_data, include_timestamp=include_timestamp) - yield (row_key, curr_row_dict) - except StopIteration: - break - - def put(self, row, data, timestamp=None, wal=_WAL_SENTINEL): - """Insert data into a row in this table. - - .. note:: - - This method will send a request with a single "put" mutation. - In many situations, :meth:`batch` is a more appropriate - method to manipulate data since it helps combine many mutations - into a single request. - - :type row: str - :param row: The row key where the mutation will be "put". - - :type data: dict - :param data: Dictionary containing the data to be inserted. The keys - are columns names (of the form ``fam:col``) and the values - are strings (bytes) to be stored in those columns. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch) that the mutation will be applied at. - - :type wal: object - :param wal: Unused parameter (to be passed to a created batch). - Provided for compatibility with HappyBase, but irrelevant - for Cloud Bigtable since it does not have a Write Ahead - Log. - """ - with self.batch(timestamp=timestamp, wal=wal) as batch: - batch.put(row, data) - - def delete(self, row, columns=None, timestamp=None, wal=_WAL_SENTINEL): - """Delete data from a row in this table. - - This method deletes the entire ``row`` if ``columns`` is not - specified. - - .. note:: - - This method will send a request with a single delete mutation. - In many situations, :meth:`batch` is a more appropriate - method to manipulate data since it helps combine many mutations - into a single request. - - :type row: str - :param row: The row key where the delete will occur. - - :type columns: list - :param columns: (Optional) Iterable containing column names (as - strings). Each column name can be either - - * an entire column family: ``fam`` or ``fam:`` - * an single column: ``fam:col`` - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch) that the mutation will be applied at. - - :type wal: object - :param wal: Unused parameter (to be passed to a created batch). - Provided for compatibility with HappyBase, but irrelevant - for Cloud Bigtable since it does not have a Write Ahead - Log. - """ - with self.batch(timestamp=timestamp, wal=wal) as batch: - batch.delete(row, columns) - - def batch(self, timestamp=None, batch_size=None, transaction=False, - wal=_WAL_SENTINEL): - """Create a new batch operation for this table. - - This method returns a new :class:`.Batch` instance that can be used - for mass data manipulation. - - :type timestamp: int - :param timestamp: (Optional) Timestamp (in milliseconds since the - epoch) that all mutations will be applied at. - - :type batch_size: int - :param batch_size: (Optional) The maximum number of mutations to allow - to accumulate before committing them. - - :type transaction: bool - :param transaction: Flag indicating if the mutations should be sent - transactionally or not. If ``transaction=True`` and - an error occurs while a :class:`Batch` is active, - then none of the accumulated mutations will be - committed. If ``batch_size`` is set, the mutation - can't be transactional. - - :type wal: object - :param wal: Unused parameter (to be passed to the created batch). - Provided for compatibility with HappyBase, but irrelevant - for Cloud Bigtable since it does not have a Write Ahead - Log. - """ - return Batch(self, timestamp=timestamp, batch_size=batch_size, - transaction=transaction, wal=wal) - - def counter_get(self, row, column): - """Retrieve the current value of a counter column. - - This method retrieves the current value of a counter column. If the - counter column does not exist, this function initializes it to ``0``. - - .. note:: - - Application code should **never** store a counter value directly; - use the atomic :meth:`counter_inc` and :meth:`counter_dec` methods - for that. - - :type row: str - :param row: Row key for the row we are getting a counter from. - - :type column: str - :param column: Column we are ``get``-ing from; of the form ``fam:col``. - - :rtype: int - :returns: Counter value (after initializing / incrementing by 0). - """ - # Don't query directly, but increment with value=0 so that the counter - # is correctly initialised if didn't exist yet. - return self.counter_inc(row, column, value=0) - - def counter_set(self, row, column, value=0): - """Set a counter column to a specific value. - - This method is provided in HappyBase, but we do not provide it here - because it defeats the purpose of using atomic increment and decrement - of a counter. - - :type row: str - :param row: Row key for the row we are setting a counter in. - - :type column: str - :param column: Column we are setting a value in; of - the form ``fam:col``. - - :type value: int - :param value: Value to set the counter to. - - :raises: :class:`NotImplementedError ` - always - """ - raise NotImplementedError('Table.counter_set will not be implemented. ' - 'Instead use the increment/decrement ' - 'methods along with counter_get.') - - def counter_inc(self, row, column, value=1): - """Atomically increment a counter column. - - This method atomically increments a counter column in ``row``. - If the counter column does not exist, it is automatically initialized - to ``0`` before being incremented. - - :type row: str - :param row: Row key for the row we are incrementing a counter in. - - :type column: str - :param column: Column we are incrementing a value in; of the - form ``fam:col``. - - :type value: int - :param value: Amount to increment the counter by. (If negative, - this is equivalent to decrement.) - - :rtype: int - :returns: Counter value after incrementing. - """ - row = self._low_level_table.row(row) - column_family_id, column_qualifier = column.split(':') - row.increment_cell_value(column_family_id, column_qualifier, value) - modified_cells = row.commit_modifications() - column_cells = modified_cells[column_family_id][column_qualifier] - if len(column_cells) != 1: - raise ValueError('Expected server to return one modified cell.') - bytes_value = column_cells[0][0] - int_value, = _UNPACK_I64(bytes_value) - return int_value - - def counter_dec(self, row, column, value=1): - """Atomically decrement a counter column. - - This method atomically decrements a counter column in ``row``. - If the counter column does not exist, it is automatically initialized - to ``0`` before being decremented. - - :type row: str - :param row: Row key for the row we are decrementing a counter in. - - :type column: str - :param column: Column we are decrementing a value in; of the - form ``fam:col``. - - :type value: int - :param value: Amount to decrement the counter by. (If negative, - this is equivalent to increment.) - - :rtype: int - :returns: Counter value after decrementing. - """ - return self.counter_inc(row, column, -value) diff --git a/gcloud_bigtable/happybase/test_batch.py b/gcloud_bigtable/happybase/test_batch.py deleted file mode 100644 index d3e7563..0000000 --- a/gcloud_bigtable/happybase/test_batch.py +++ /dev/null @@ -1,530 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class _SendMixin(object): - - _send_called = False - - def send(self): - self._send_called = True - - -class Test__get_column_pairs(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.batch import _get_column_pairs - return _get_column_pairs(*args, **kwargs) - - def test_it(self): - columns = ['cf1', 'cf2:', 'cf3::', 'cf3:name1', 'cf3:name2'] - result = self._callFUT(columns) - expected_result = [ - ['cf1', None], - ['cf2', None], - ['cf3', None], - ['cf3', 'name1'], - ['cf3', 'name2'], - ] - self.assertEqual(result, expected_result) - - def test_bad_column(self): - columns = ['a:b:c'] - with self.assertRaises(ValueError): - self._callFUT(columns) - - def test_bad_column_type(self): - columns = [None] - with self.assertRaises(AttributeError): - self._callFUT(columns) - - def test_bad_columns_var(self): - columns = None - with self.assertRaises(TypeError): - self._callFUT(columns) - - def test_column_family_with_require_qualifier(self): - columns = ['a:'] - with self.assertRaises(ValueError): - self._callFUT(columns, require_qualifier=True) - - -class TestBatch(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.batch import Batch - return Batch - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - table = object() - batch = self._makeOne(table) - self.assertEqual(batch._table, table) - self.assertEqual(batch._batch_size, None) - self.assertEqual(batch._timestamp, None) - self.assertEqual(batch._delete_range, None) - self.assertEqual(batch._transaction, False) - - def test_constructor_explicit(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp - from gcloud_bigtable.row import TimestampRange - - table = object() - timestamp = 144185290431 - batch_size = 42 - transaction = False # Must be False when batch_size is non-null - - batch = self._makeOne(table, timestamp=timestamp, - batch_size=batch_size, transaction=transaction) - self.assertEqual(batch._table, table) - self.assertEqual(batch._batch_size, batch_size) - self.assertEqual(batch._timestamp, - _microseconds_to_timestamp(1000 * timestamp)) - - next_timestamp = _microseconds_to_timestamp(1000 * (timestamp + 1)) - time_range = TimestampRange(end=next_timestamp) - self.assertEqual(batch._delete_range, time_range) - self.assertEqual(batch._transaction, transaction) - - def test_constructor_with_non_default_wal(self): - table = object() - wal = object() - with self.assertRaises(ValueError): - self._makeOne(table, wal=wal) - - def test_constructor_with_non_positive_batch_size(self): - table = object() - batch_size = -10 - with self.assertRaises(ValueError): - self._makeOne(table, batch_size=batch_size) - batch_size = 0 - with self.assertRaises(ValueError): - self._makeOne(table, batch_size=batch_size) - - def test_constructor_with_batch_size_and_transactional(self): - table = object() - batch_size = 1 - transaction = True - with self.assertRaises(TypeError): - self._makeOne(table, batch_size=batch_size, - transaction=transaction) - - def test_send(self): - table = object() - batch = self._makeOne(table) - - batch._row_map = row_map = _MockRowMap() - row_map['row-key1'] = row1 = _MockRow() - row_map['row-key2'] = row2 = _MockRow() - batch._mutation_count = 1337 - - self.assertEqual(row_map.clear_count, 0) - self.assertEqual(row1.commits, 0) - self.assertEqual(row2.commits, 0) - self.assertNotEqual(batch._mutation_count, 0) - self.assertNotEqual(row_map, {}) - - batch.send() - self.assertEqual(row_map.clear_count, 1) - self.assertEqual(row1.commits, 1) - self.assertEqual(row2.commits, 1) - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(row_map, {}) - - def test__try_send_no_batch_size(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table) - - self.assertEqual(batch._batch_size, None) - self.assertFalse(batch._send_called) - batch._try_send() - self.assertFalse(batch._send_called) - - def test__try_send_too_few_mutations(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch_size = 10 - batch = BatchWithSend(table, batch_size=batch_size) - - self.assertEqual(batch._batch_size, batch_size) - self.assertFalse(batch._send_called) - mutation_count = 2 - batch._mutation_count = mutation_count - self.assertTrue(mutation_count < batch_size) - batch._try_send() - self.assertFalse(batch._send_called) - - def test__try_send_actual_send(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch_size = 10 - batch = BatchWithSend(table, batch_size=batch_size) - - self.assertEqual(batch._batch_size, batch_size) - self.assertFalse(batch._send_called) - mutation_count = 12 - batch._mutation_count = mutation_count - self.assertTrue(mutation_count > batch_size) - batch._try_send() - self.assertTrue(batch._send_called) - - def test__get_row_exists(self): - table = object() - batch = self._makeOne(table) - - row_key = 'row-key' - row_obj = object() - batch._row_map[row_key] = row_obj - result = batch._get_row(row_key) - self.assertEqual(result, row_obj) - - def test__get_row_create_new(self): - # Make mock batch and make sure we can create a low-level table. - low_level_table = _MockLowLevelTable() - table = _MockTable(low_level_table) - batch = self._makeOne(table) - - # Make sure row map is empty. - self.assertEqual(batch._row_map, {}) - - # Customize/capture mock table creation. - low_level_table.mock_row = mock_row = object() - - # Actually get the row (which creates a row via a low-level table). - row_key = 'row-key' - result = batch._get_row(row_key) - self.assertEqual(result, mock_row) - - # Check all the things that were constructed. - self.assertEqual(low_level_table.rows_made, [row_key]) - # Check how the batch was updated. - self.assertEqual(batch._row_map, {row_key: mock_row}) - - def test_put_bad_wal(self): - from gcloud_bigtable.happybase.batch import _WAL_SENTINEL - - table = object() - batch = self._makeOne(table) - - row = 'row-key' - data = {} - wal = None - - self.assertNotEqual(wal, _WAL_SENTINEL) - with self.assertRaises(ValueError): - batch.put(row, data, wal=wal) - - def test_put(self): - import operator - - table = object() - batch = self._makeOne(table) - batch._timestamp = timestamp = object() - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - col1_fam = 'cf1' - col1_qual = 'qual1' - value1 = 'value1' - col2_fam = 'cf2' - col2_qual = 'qual2' - value2 = 'value2' - data = {col1_fam + ':' + col1_qual: value1, - col2_fam + ':' + col2_qual: value2} - - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(row.set_cell_calls, []) - batch.put(row_key, data) - self.assertEqual(batch._mutation_count, 2) - # Since the calls depend on data.keys(), the order - # is non-deterministic. - first_elt = operator.itemgetter(0) - ordered_calls = sorted(row.set_cell_calls, key=first_elt) - - cell1_args = (col1_fam, col1_qual, value1) - cell1_kwargs = {'timestamp': timestamp} - cell2_args = (col2_fam, col2_qual, value2) - cell2_kwargs = {'timestamp': timestamp} - self.assertEqual(ordered_calls, [ - (cell1_args, cell1_kwargs), - (cell2_args, cell2_kwargs), - ]) - - def test_put_call_try_send(self): - klass = self._getTargetClass() - - class CallTrySend(klass): - - try_send_calls = 0 - - def _try_send(self): - self.try_send_calls += 1 - - table = object() - batch = CallTrySend(table) - - row_key = 'row-key' - batch._row_map[row_key] = _MockRow() - - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 0) - # No data so that nothing happens - batch.put(row_key, data={}) - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 1) - - def _delete_columns_test_helper(self, time_range=None): - table = object() - batch = self._makeOne(table) - batch._delete_range = time_range - - col1_fam = 'cf1' - col2_fam = 'cf2' - col2_qual = 'col-name' - columns = [col1_fam + ':', col2_fam + ':' + col2_qual] - row_object = _MockRow() - - batch._delete_columns(columns, row_object) - self.assertEqual(row_object.commits, 0) - - cell_deleted_args = (col2_fam, col2_qual) - cell_deleted_kwargs = {'time_range': time_range} - self.assertEqual(row_object.delete_cell_calls, - [(cell_deleted_args, cell_deleted_kwargs)]) - fam_deleted_args = (col1_fam,) - fam_deleted_kwargs = {'columns': row_object.ALL_COLUMNS} - self.assertEqual(row_object.delete_cells_calls, - [(fam_deleted_args, fam_deleted_kwargs)]) - - def test__delete_columns(self): - self._delete_columns_test_helper() - - def test__delete_columns_w_time_and_col_fam(self): - time_range = object() - with self.assertRaises(ValueError): - self._delete_columns_test_helper(time_range=time_range) - - def test_delete_bad_wal(self): - from gcloud_bigtable.happybase.batch import _WAL_SENTINEL - - table = object() - batch = self._makeOne(table) - - row = 'row-key' - columns = [] - wal = None - - self.assertNotEqual(wal, _WAL_SENTINEL) - with self.assertRaises(ValueError): - batch.delete(row, columns=columns, wal=wal) - - def test_delete_entire_row(self): - table = object() - batch = self._makeOne(table) - - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - self.assertEqual(row.deletes, 0) - self.assertEqual(batch._mutation_count, 0) - batch.delete(row_key, columns=None) - self.assertEqual(row.deletes, 1) - self.assertEqual(batch._mutation_count, 1) - - def test_delete_entire_row_with_ts(self): - table = object() - batch = self._makeOne(table) - batch._delete_range = object() - - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - self.assertEqual(row.deletes, 0) - self.assertEqual(batch._mutation_count, 0) - with self.assertRaises(ValueError): - batch.delete(row_key, columns=None) - self.assertEqual(row.deletes, 0) - self.assertEqual(batch._mutation_count, 0) - - def test_delete_call_try_send(self): - klass = self._getTargetClass() - - class CallTrySend(klass): - - try_send_calls = 0 - - def _try_send(self): - self.try_send_calls += 1 - - table = object() - batch = CallTrySend(table) - - row_key = 'row-key' - batch._row_map[row_key] = _MockRow() - - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 0) - # No columns so that nothing happens - batch.delete(row_key, columns=[]) - self.assertEqual(batch._mutation_count, 0) - self.assertEqual(batch.try_send_calls, 1) - - def test_delete_some_columns(self): - table = object() - batch = self._makeOne(table) - - row_key = 'row-key' - batch._row_map[row_key] = row = _MockRow() - - self.assertEqual(batch._mutation_count, 0) - - col1_fam = 'cf1' - col2_fam = 'cf2' - col2_qual = 'col-name' - columns = [col1_fam + ':', col2_fam + ':' + col2_qual] - batch.delete(row_key, columns=columns) - - self.assertEqual(batch._mutation_count, 2) - cell_deleted_args = (col2_fam, col2_qual) - cell_deleted_kwargs = {'time_range': None} - self.assertEqual(row.delete_cell_calls, - [(cell_deleted_args, cell_deleted_kwargs)]) - fam_deleted_args = (col1_fam,) - fam_deleted_kwargs = {'columns': row.ALL_COLUMNS} - self.assertEqual(row.delete_cells_calls, - [(fam_deleted_args, fam_deleted_kwargs)]) - - def test_context_manager(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table) - self.assertFalse(batch._send_called) - - with batch: - pass - - self.assertTrue(batch._send_called) - - def test_context_manager_with_exception_non_transactional(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table) - self.assertFalse(batch._send_called) - - with self.assertRaises(ValueError): - with batch: - raise ValueError('Something bad happened') - - self.assertTrue(batch._send_called) - - def test_context_manager_with_exception_transactional(self): - klass = self._getTargetClass() - - class BatchWithSend(_SendMixin, klass): - pass - - table = object() - batch = BatchWithSend(table, transaction=True) - self.assertFalse(batch._send_called) - - with self.assertRaises(ValueError): - with batch: - raise ValueError('Something bad happened') - - self.assertFalse(batch._send_called) - - # Just to make sure send() actually works (and to make cover happy). - batch.send() - self.assertTrue(batch._send_called) - - -class _MockRowMap(dict): - - clear_count = 0 - - def clear(self): - self.clear_count += 1 - super(_MockRowMap, self).clear() - - -class _MockRow(object): - - ALL_COLUMNS = object() - - def __init__(self): - self.commits = 0 - self.deletes = 0 - self.set_cell_calls = [] - self.delete_cell_calls = [] - self.delete_cells_calls = [] - - def commit(self): - self.commits += 1 - - def delete(self): - self.deletes += 1 - - def set_cell(self, *args, **kwargs): - self.set_cell_calls.append((args, kwargs)) - - def delete_cell(self, *args, **kwargs): - self.delete_cell_calls.append((args, kwargs)) - - def delete_cells(self, *args, **kwargs): - self.delete_cells_calls.append((args, kwargs)) - - -class _MockTable(object): - - def __init__(self, low_level_table): - self._low_level_table = low_level_table - - -class _MockLowLevelTable(object): - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.rows_made = [] - self.mock_row = None - - def row(self, row_key): - self.rows_made.append(row_key) - return self.mock_row diff --git a/gcloud_bigtable/happybase/test_connection.py b/gcloud_bigtable/happybase/test_connection.py deleted file mode 100644 index 397ecea..0000000 --- a/gcloud_bigtable/happybase/test_connection.py +++ /dev/null @@ -1,579 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class Test__get_cluster(unittest2.TestCase): - - def _callFUT(self, timeout=None): - from gcloud_bigtable.happybase.connection import _get_cluster - return _get_cluster(timeout=timeout) - - def _helper(self, timeout=None, clusters=(), failed_zones=()): - from functools import partial - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - client_with_clusters = partial(_Client, clusters=clusters, - failed_zones=failed_zones) - with _Monkey(MUT, Client=client_with_clusters): - result = self._callFUT(timeout=timeout) - - # If we've reached this point, then _callFUT didn't fail, so we know - # there is exactly one cluster. - cluster, = clusters - self.assertEqual(result, cluster) - client = cluster.client - self.assertEqual(client.args, ()) - expected_kwargs = {'admin': True} - if timeout is not None: - expected_kwargs['timeout_seconds'] = timeout / 1000.0 - self.assertEqual(client.kwargs, expected_kwargs) - self.assertEqual(client.start_calls, 1) - self.assertEqual(client.stop_calls, 1) - - def test_default(self): - cluster = _Cluster() - self._helper(clusters=[cluster]) - - def test_with_timeout(self): - cluster = _Cluster() - self._helper(timeout=2103, clusters=[cluster]) - - def test_with_no_clusters(self): - with self.assertRaises(ValueError): - self._helper() - - def test_with_too_many_clusters(self): - clusters = [_Cluster(), _Cluster()] - with self.assertRaises(ValueError): - self._helper(clusters=clusters) - - def test_with_failed_zones(self): - cluster = _Cluster() - failed_zone = 'us-central1-c' - with self.assertRaises(ValueError): - self._helper(clusters=[cluster], - failed_zones=[failed_zone]) - - -class Test__parse_family_option(unittest2.TestCase): - - def _callFUT(self, option): - from gcloud_bigtable.happybase.connection import _parse_family_option - return _parse_family_option(option) - - def test_dictionary_no_keys(self): - option = {} - result = self._callFUT(option) - self.assertEqual(result, None) - - def test_null(self): - option = None - result = self._callFUT(option) - self.assertEqual(result, None) - - def test_dictionary_bad_key(self): - option = {'badkey': None} - with self.assertRaises(ValueError): - self._callFUT(option) - - def test_dictionary_versions_key(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - - versions = 42 - option = {'max_versions': versions} - result = self._callFUT(option) - - gc_rule = GarbageCollectionRule(max_num_versions=versions) - self.assertEqual(result, gc_rule) - - def test_dictionary_ttl_key(self): - import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - - time_to_live = 24 * 60 * 60 - max_age = datetime.timedelta(days=1) - option = {'time_to_live': time_to_live} - result = self._callFUT(option) - - gc_rule = GarbageCollectionRule(max_age=max_age) - self.assertEqual(result, gc_rule) - - def test_dictionary_both_keys(self): - import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - - versions = 42 - time_to_live = 24 * 60 * 60 - option = { - 'max_versions': versions, - 'time_to_live': time_to_live, - } - result = self._callFUT(option) - - max_age = datetime.timedelta(days=1) - # NOTE: This relies on the order of the rules in the method we are - # calling matching this order here. - gc_rule1 = GarbageCollectionRule(max_age=max_age) - gc_rule2 = GarbageCollectionRule(max_num_versions=versions) - gc_rule = GarbageCollectionRuleIntersection( - rules=[gc_rule1, gc_rule2]) - self.assertEqual(result, gc_rule) - - def test_non_dictionary(self): - option = object() - self.assertFalse(isinstance(option, dict)) - result = self._callFUT(option) - self.assertEqual(result, option) - - -class TestConnection(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.connection import Connection - return Connection - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - cluster = _Cluster() # Avoid implicit environ check. - self.assertEqual(cluster.client.start_calls, 0) - connection = self._makeOne(cluster=cluster) - self.assertEqual(cluster.client.start_calls, 1) - self.assertEqual(cluster.client.stop_calls, 0) - - self.assertEqual(connection._cluster, cluster) - self.assertEqual(connection.table_prefix, None) - self.assertEqual(connection.table_prefix_separator, '_') - - def test_constructor_no_autoconnect(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(connection.table_prefix, None) - self.assertEqual(connection.table_prefix_separator, '_') - - def test_constructor_missing_cluster(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - cluster = _Cluster() - timeout = object() - mock_get_cluster = _MockCalled(cluster) - with _Monkey(MUT, _get_cluster=mock_get_cluster): - connection = self._makeOne(autoconnect=False, cluster=None, - timeout=timeout) - self.assertEqual(connection.table_prefix, None) - self.assertEqual(connection.table_prefix_separator, '_') - self.assertEqual(connection._cluster, cluster) - - mock_get_cluster.check_called(self, [()], [{'timeout': timeout}]) - - def test_constructor_explicit(self): - table_prefix = 'table-prefix' - table_prefix_separator = 'sep' - cluster_copy = _Cluster() - cluster = _Cluster(copies=[cluster_copy]) - - connection = self._makeOne( - autoconnect=False, - table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - self.assertEqual(connection.table_prefix, table_prefix) - self.assertEqual(connection.table_prefix_separator, - table_prefix_separator) - self.assertEqual(connection._cluster, cluster_copy) - - def test_constructor_non_string_prefix(self): - table_prefix = object() - - with self.assertRaises(TypeError): - self._makeOne(autoconnect=False, - table_prefix=table_prefix) - - def test_constructor_non_string_prefix_separator(self): - table_prefix_separator = object() - - with self.assertRaises(TypeError): - self._makeOne(autoconnect=False, - table_prefix_separator=table_prefix_separator) - - def test_constructor_with_host(self): - with self.assertRaises(ValueError): - self._makeOne(host=object()) - - def test_constructor_with_port(self): - with self.assertRaises(ValueError): - self._makeOne(port=object()) - - def test_constructor_with_compat(self): - with self.assertRaises(ValueError): - self._makeOne(compat=object()) - - def test_constructor_with_transport(self): - with self.assertRaises(ValueError): - self._makeOne(transport=object()) - - def test_constructor_with_protocol(self): - with self.assertRaises(ValueError): - self._makeOne(protocol=object()) - - def test_constructor_with_timeout_and_cluster(self): - with self.assertRaises(ValueError): - self._makeOne(cluster=object(), timeout=object()) - - def test_open(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster.client.start_calls, 0) - connection.open() - self.assertEqual(cluster.client.start_calls, 1) - self.assertEqual(cluster.client.stop_calls, 0) - - def test_close(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster.client.stop_calls, 0) - connection.close() - self.assertEqual(cluster.client.stop_calls, 1) - self.assertEqual(cluster.client.start_calls, 0) - - def test___del__good_initialization(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - self.assertEqual(cluster.client.stop_calls, 0) - connection.__del__() - self.assertEqual(cluster.client.stop_calls, 1) - - def test___del__bad_initialization(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - # Fake that initialization failed. - del connection._initialized - - self.assertEqual(cluster.client.stop_calls, 0) - connection.__del__() - self.assertEqual(cluster.client.stop_calls, 0) - - def test__table_name_with_prefix_set(self): - table_prefix = 'table-prefix' - table_prefix_separator = '<>' - cluster = _Cluster() - - connection = self._makeOne( - autoconnect=False, - table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - - name = 'some-name' - prefixed = connection._table_name(name) - self.assertEqual(prefixed, - table_prefix + table_prefix_separator + name) - - def test__table_name_with_no_prefix_set(self): - cluster = _Cluster() - connection = self._makeOne(autoconnect=False, - cluster=cluster) - - name = 'some-name' - prefixed = connection._table_name(name) - self.assertEqual(prefixed, name) - - def test_table_factory(self): - from gcloud_bigtable.happybase.table import Table - - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - table = connection.table(name) - - self.assertTrue(isinstance(table, Table)) - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - - def _table_factory_prefix_helper(self, use_prefix=True): - from gcloud_bigtable.happybase.table import Table - - cluster = _Cluster() # Avoid implicit environ check. - table_prefix = 'table-prefix' - table_prefix_separator = '<>' - connection = self._makeOne( - autoconnect=False, table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - - name = 'table-name' - table = connection.table(name, use_prefix=use_prefix) - - self.assertTrue(isinstance(table, Table)) - prefixed_name = table_prefix + table_prefix_separator + name - if use_prefix: - self.assertEqual(table.name, prefixed_name) - else: - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - - def test_table_factory_with_prefix(self): - self._table_factory_prefix_helper(use_prefix=True) - - def test_table_factory_with_ignored_prefix(self): - self._table_factory_prefix_helper(use_prefix=False) - - def test_tables(self): - from gcloud_bigtable.table import Table - - table_name1 = 'table-name1' - table_name2 = 'table-name2' - cluster = _Cluster(list_tables_result=[ - Table(table_name1, None), - Table(table_name2, None), - ]) - connection = self._makeOne(autoconnect=False, cluster=cluster) - result = connection.tables() - self.assertEqual(result, [table_name1, table_name2]) - - def test_tables_with_prefix(self): - from gcloud_bigtable.table import Table - - table_prefix = 'prefix' - table_prefix_separator = '<>' - unprefixed_table_name1 = 'table-name1' - - table_name1 = (table_prefix + table_prefix_separator + - unprefixed_table_name1) - table_name2 = 'table-name2' - cluster = _Cluster(list_tables_result=[ - Table(table_name1, None), - Table(table_name2, None), - ]) - connection = self._makeOne( - autoconnect=False, cluster=cluster, table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator) - result = connection.tables() - self.assertEqual(result, [unprefixed_table_name1]) - - def test_create_table(self): - import operator - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - mock_gc_rule = object() - mock_parse_family_option = _MockCalled(mock_gc_rule) - - name = 'table-name' - col_fam1 = 'cf1' - col_fam_option1 = object() - col_fam2 = 'cf2' - col_fam_option2 = object() - families = { - col_fam1: col_fam_option1, - # A trailing colon is also allowed. - col_fam2 + ':': col_fam_option2, - } - table_instances = [] - col_fam_instances = [] - with _Monkey(MUT, _LowLevelTable=_MockLowLevelTable, - _parse_family_option=mock_parse_family_option): - _MockLowLevelTable._instances = table_instances - _MockLowLevelColumnFamily._instances = col_fam_instances - connection.create_table(name, families) - - # Just one table would have been created. - table_instance, = table_instances - self.assertEqual(table_instance.args, (name, cluster)) - self.assertEqual(table_instance.kwargs, {}) - self.assertEqual(table_instance.create_calls, 1) - - # Check if our mock was called twice, but we don't know the order. - mock_called = mock_parse_family_option.called_args - self.assertEqual(len(mock_called), 2) - self.assertEqual([len(args) for args in mock_called], [1, 1]) - self.assertEqual(set(mock_called[0] + mock_called[1]), - set([col_fam_option1, col_fam_option2])) - - # We expect two column family instances created, but don't know the - # order due to non-deterministic dict.items(). - col_fam_instances.sort(key=operator.attrgetter('column_family_id')) - self.assertEqual(col_fam_instances[0].column_family_id, col_fam1) - self.assertEqual(col_fam_instances[0].gc_rule, mock_gc_rule) - self.assertEqual(col_fam_instances[0].create_calls, 1) - self.assertEqual(col_fam_instances[1].column_family_id, col_fam2) - self.assertEqual(col_fam_instances[1].gc_rule, mock_gc_rule) - self.assertEqual(col_fam_instances[1].create_calls, 1) - - def test_create_table_bad_type(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - families = None - with self.assertRaises(TypeError): - connection.create_table(name, families) - - def test_create_table_bad_value(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - families = {} - with self.assertRaises(ValueError): - connection.create_table(name, families) - - def test_delete_table(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import connection as MUT - - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - instances = [] - with _Monkey(MUT, _LowLevelTable=_MockLowLevelTable): - _MockLowLevelTable._instances = instances - connection.delete_table(name) - - # Just one table would have been created. - table_instance, = instances - self.assertEqual(table_instance.args, (name, cluster)) - self.assertEqual(table_instance.kwargs, {}) - self.assertEqual(table_instance.delete_calls, 1) - - def test_delete_table_disable(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - name = 'table-name' - with self.assertRaises(ValueError): - connection.delete_table(name, disable=True) - - def test_enable_table(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - with self.assertRaises(NotImplementedError): - connection.enable_table(name) - - def test_disable_table(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - with self.assertRaises(NotImplementedError): - connection.disable_table(name) - - def test_is_table_enabled(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - with self.assertRaises(NotImplementedError): - connection.is_table_enabled(name) - - def test_compact_table(self): - cluster = _Cluster() # Avoid implicit environ check. - connection = self._makeOne(autoconnect=False, cluster=cluster) - - name = 'table-name' - major = True - with self.assertRaises(NotImplementedError): - connection.compact_table(name, major=major) - - -class _Client(object): - - def __init__(self, *args, **kwargs): - self.clusters = kwargs.pop('clusters', []) - for cluster in self.clusters: - cluster.client = self - self.failed_zones = kwargs.pop('failed_zones', []) - self.args = args - self.kwargs = kwargs - self.start_calls = 0 - self.stop_calls = 0 - - def start(self): - self.start_calls += 1 - - def stop(self): - self.stop_calls += 1 - - def list_clusters(self): - return self.clusters, self.failed_zones - - -class _Cluster(object): - - def __init__(self, copies=(), list_tables_result=()): - self.copies = list(copies) - # Included to support Connection.__del__ - self.client = _Client() - self.list_tables_result = list_tables_result - - def copy(self): - if self.copies: - result = self.copies[0] - self.copies[:] = self.copies[1:] - return result - else: - return self - - def list_tables(self): - return self.list_tables_result - - -class _MockLowLevelTable(object): - - _instances = [] - - def __init__(self, *args, **kwargs): - self._instances.append(self) - self.args = args - self.kwargs = kwargs - self.delete_calls = 0 - self.create_calls = 0 - - def delete(self): - self.delete_calls += 1 - - def create(self): - self.create_calls += 1 - - def column_family(self, column_family_id, gc_rule=None): - return _MockLowLevelColumnFamily(column_family_id, gc_rule=gc_rule) - - -class _MockLowLevelColumnFamily(object): - - _instances = [] - - def __init__(self, column_family_id, gc_rule=None): - self._instances.append(self) - self.column_family_id = column_family_id - self.gc_rule = gc_rule - self.create_calls = 0 - - def create(self): - self.create_calls += 1 diff --git a/gcloud_bigtable/happybase/test_pool.py b/gcloud_bigtable/happybase/test_pool.py deleted file mode 100644 index 6791c57..0000000 --- a/gcloud_bigtable/happybase/test_pool.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestConnectionPool(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.pool import ConnectionPool - return ConnectionPool - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - import six - import threading - from gcloud_bigtable.happybase.connection import Connection - - size = 11 - cluster_copy = _Cluster() - all_copies = [cluster_copy] * size - cluster = _Cluster(copies=all_copies) # Avoid implicit environ check. - pool = self._makeOne(size, cluster=cluster) - - self.assertTrue(isinstance(pool._lock, type(threading.Lock()))) - self.assertTrue(isinstance(pool._thread_connections, threading.local)) - self.assertEqual(pool._thread_connections.__dict__, {}) - - queue = pool._queue - self.assertTrue(isinstance(queue, six.moves.queue.LifoQueue)) - self.assertTrue(queue.full()) - self.assertEqual(queue.maxsize, size) - for connection in queue.queue: - self.assertTrue(isinstance(connection, Connection)) - self.assertTrue(connection._cluster is cluster_copy) - - def test_constructor_passes_kwargs(self): - table_prefix = 'foo' - table_prefix_separator = '<>' - cluster = _Cluster() # Avoid implicit environ check. - - size = 1 - pool = self._makeOne(size, table_prefix=table_prefix, - table_prefix_separator=table_prefix_separator, - cluster=cluster) - - for connection in pool._queue.queue: - self.assertEqual(connection.table_prefix, table_prefix) - self.assertEqual(connection.table_prefix_separator, - table_prefix_separator) - - def test_constructor_ignores_autoconnect(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase.connection import Connection - from gcloud_bigtable.happybase import pool as MUT - - class ConnectionWithOpen(Connection): - - _open_called = False - - def open(self): - self._open_called = True - - # First make sure the custom Connection class does as expected. - cluster_copy1 = _Cluster() - cluster_copy2 = _Cluster() - cluster_copy3 = _Cluster() - cluster = _Cluster( - copies=[cluster_copy1, cluster_copy2, cluster_copy3]) - connection = ConnectionWithOpen(autoconnect=False, cluster=cluster) - self.assertFalse(connection._open_called) - self.assertTrue(connection._cluster is cluster_copy1) - connection = ConnectionWithOpen(autoconnect=True, cluster=cluster) - self.assertTrue(connection._open_called) - self.assertTrue(connection._cluster is cluster_copy2) - - # Then make sure autoconnect=True is ignored in a pool. - size = 1 - with _Monkey(MUT, Connection=ConnectionWithOpen): - pool = self._makeOne(size, autoconnect=True, cluster=cluster) - - for connection in pool._queue.queue: - self.assertTrue(isinstance(connection, ConnectionWithOpen)) - self.assertTrue(connection._cluster is cluster_copy3) - self.assertFalse(connection._open_called) - - def test_constructor_infers_cluster(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase.connection import Connection - from gcloud_bigtable.happybase import pool as MUT - - size = 1 - cluster_copy = _Cluster() - all_copies = [cluster_copy] * size - cluster = _Cluster(copies=all_copies) - - mock_get_cluster = _MockCalled(cluster) - with _Monkey(MUT, _get_cluster=mock_get_cluster): - pool = self._makeOne(size) - - for connection in pool._queue.queue: - self.assertTrue(isinstance(connection, Connection)) - # We know that the Connection() constructor will - # call cluster.copy(). - self.assertTrue(connection._cluster is cluster_copy) - - mock_get_cluster.check_called(self, [()], [{'timeout': None}]) - - def test_constructor_non_integer_size(self): - size = None - with self.assertRaises(TypeError): - self._makeOne(size) - - def test_constructor_non_positive_size(self): - size = -10 - with self.assertRaises(ValueError): - self._makeOne(size) - size = 0 - with self.assertRaises(ValueError): - self._makeOne(size) - - def _makeOneWithMockQueue(self, queue_return): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import pool as MUT - - # We are going to use a fake queue, so we don't want any connections - # or clusters to be created in the constructor. - size = -1 - cluster = object() - with _Monkey(MUT, _MIN_POOL_SIZE=size): - pool = self._makeOne(size, cluster=cluster) - - pool._queue = _Queue(queue_return) - return pool - - def test__acquire_connection(self): - queue_return = object() - pool = self._makeOneWithMockQueue(queue_return) - - timeout = 432 - connection = pool._acquire_connection(timeout=timeout) - self.assertTrue(connection is queue_return) - self.assertEqual(pool._queue._get_calls, [(True, timeout)]) - self.assertEqual(pool._queue._put_calls, []) - - def test__acquire_connection_failure(self): - from gcloud_bigtable.happybase.pool import NoConnectionsAvailable - - pool = self._makeOneWithMockQueue(None) - timeout = 1027 - with self.assertRaises(NoConnectionsAvailable): - pool._acquire_connection(timeout=timeout) - self.assertEqual(pool._queue._get_calls, [(True, timeout)]) - self.assertEqual(pool._queue._put_calls, []) - - def test_connection_is_context_manager(self): - import contextlib - - queue_return = _Connection() - pool = self._makeOneWithMockQueue(queue_return) - cnxn_context = pool.connection() - self.assertTrue(isinstance(cnxn_context, - contextlib.GeneratorContextManager)) - - def test_connection_no_current_cnxn(self): - queue_return = _Connection() - pool = self._makeOneWithMockQueue(queue_return) - timeout = 55 - - self.assertFalse(hasattr(pool._thread_connections, 'current')) - with pool.connection(timeout=timeout) as connection: - self.assertEqual(pool._thread_connections.current, queue_return) - self.assertTrue(connection is queue_return) - self.assertFalse(hasattr(pool._thread_connections, 'current')) - - self.assertEqual(pool._queue._get_calls, [(True, timeout)]) - self.assertEqual(pool._queue._put_calls, - [(queue_return, None, None)]) - - def test_connection_with_current_cnxn(self): - current_cnxn = _Connection() - queue_return = _Connection() - pool = self._makeOneWithMockQueue(queue_return) - pool._thread_connections.current = current_cnxn - timeout = 8001 - - with pool.connection(timeout=timeout) as connection: - self.assertTrue(connection is current_cnxn) - - self.assertEqual(pool._queue._get_calls, []) - self.assertEqual(pool._queue._put_calls, []) - self.assertEqual(pool._thread_connections.current, current_cnxn) - - -class _Client(object): - - def __init__(self): - self.stop_calls = 0 - - def stop(self): - self.stop_calls += 1 - - -class _Connection(object): - - def open(self): - pass - - -class _Cluster(object): - - def __init__(self, copies=()): - self.copies = list(copies) - # Included to support Connection.__del__ - self.client = _Client() - - def copy(self): - if self.copies: - result = self.copies[0] - self.copies[:] = self.copies[1:] - return result - else: - return self - - -class _Queue(object): - - def __init__(self, result=None): - self.result = result - self._get_calls = [] - self._put_calls = [] - - def get(self, block=None, timeout=None): - self._get_calls.append((block, timeout)) - if self.result is None: - import six - raise six.moves.queue.Empty - else: - return self.result - - def put(self, item, block=None, timeout=None): - self._put_calls.append((item, block, timeout)) diff --git a/gcloud_bigtable/happybase/test_table.py b/gcloud_bigtable/happybase/test_table.py deleted file mode 100644 index 94eb784..0000000 --- a/gcloud_bigtable/happybase/test_table.py +++ /dev/null @@ -1,1304 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class Test_make_row(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import make_row - return make_row(*args, **kwargs) - - def test_it(self): - with self.assertRaises(NotImplementedError): - self._callFUT({}, False) - - -class Test_make_ordered_row(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import make_ordered_row - return make_ordered_row(*args, **kwargs) - - def test_it(self): - with self.assertRaises(NotImplementedError): - self._callFUT([], False) - - -class Test__gc_rule_to_dict(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _gc_rule_to_dict - return _gc_rule_to_dict(*args, **kwargs) - - def test_with_null(self): - gc_rule = None - result = self._callFUT(gc_rule) - self.assertEqual(result, {}) - - def test_with_max_versions(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - - max_versions = 2 - gc_rule = GarbageCollectionRule(max_num_versions=max_versions) - result = self._callFUT(gc_rule) - expected_result = {'max_versions': max_versions} - self.assertEqual(result, expected_result) - - def test_with_max_age(self): - import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - - time_to_live = 101 - max_age = datetime.timedelta(seconds=time_to_live) - gc_rule = GarbageCollectionRule(max_age=max_age) - result = self._callFUT(gc_rule) - expected_result = {'time_to_live': time_to_live} - self.assertEqual(result, expected_result) - - def test_with_non_gc_rule(self): - gc_rule = object() - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_gc_rule_union(self): - from gcloud_bigtable.column_family import GarbageCollectionRuleUnion - - gc_rule = GarbageCollectionRuleUnion() - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_intersection_other_than_two(self): - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - - gc_rule = GarbageCollectionRuleIntersection(rules=[]) - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_intersection_two_max_num_versions(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - - rule1 = GarbageCollectionRule(max_num_versions=1) - rule2 = GarbageCollectionRule(max_num_versions=2) - gc_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - def test_with_intersection_two_rules(self): - import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - - time_to_live = 101 - max_age = datetime.timedelta(seconds=time_to_live) - rule1 = GarbageCollectionRule(max_age=max_age) - max_versions = 2 - rule2 = GarbageCollectionRule(max_num_versions=max_versions) - gc_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) - result = self._callFUT(gc_rule) - expected_result = { - 'max_versions': max_versions, - 'time_to_live': time_to_live, - } - self.assertEqual(result, expected_result) - - def test_with_intersection_two_nested_rules(self): - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - - rule1 = GarbageCollectionRuleIntersection(rules=[]) - rule2 = GarbageCollectionRuleIntersection(rules=[]) - gc_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) - result = self._callFUT(gc_rule) - self.assertTrue(result is gc_rule) - - -class Test__convert_to_time_range(unittest2.TestCase): - - def _callFUT(self, timestamp=None): - from gcloud_bigtable.happybase.table import _convert_to_time_range - return _convert_to_time_range(timestamp=timestamp) - - def test_null(self): - timestamp = None - result = self._callFUT(timestamp=timestamp) - self.assertEqual(result, None) - - def test_invalid_type(self): - timestamp = object() - with self.assertRaises(TypeError): - self._callFUT(timestamp=timestamp) - - def test_success(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp - from gcloud_bigtable.row import TimestampRange - - timestamp = 1441928298571 - ts_dt = _microseconds_to_timestamp(1000 * timestamp) - result = self._callFUT(timestamp=timestamp) - self.assertTrue(isinstance(result, TimestampRange)) - self.assertEqual(result.start, None) - self.assertEqual(result.end, ts_dt) - - -class Test__cells_to_pairs(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _cells_to_pairs - return _cells_to_pairs(*args, **kwargs) - - def test_without_timestamp(self): - from gcloud_bigtable.row_data import Cell - - value1 = 'foo' - cell1 = Cell(value=value1, timestamp=None) - value2 = 'bar' - cell2 = Cell(value=value2, timestamp=None) - - result = self._callFUT([cell1, cell2]) - self.assertEqual(result, [value1, value2]) - - def test_with_timestamp(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp - from gcloud_bigtable.row_data import Cell - - value1 = 'foo' - ts1_millis = 1221934570148 - ts1 = _microseconds_to_timestamp(ts1_millis * 1000) - cell1 = Cell(value=value1, timestamp=ts1) - - value2 = 'bar' - ts2_millis = 1221955575548 - ts2 = _microseconds_to_timestamp(ts2_millis * 1000) - cell2 = Cell(value=value2, timestamp=ts2) - - result = self._callFUT([cell1, cell2], include_timestamp=True) - self.assertEqual(result, - [(value1, ts1_millis), (value2, ts2_millis)]) - - -class Test__filter_chain_helper(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _filter_chain_helper - return _filter_chain_helper(*args, **kwargs) - - def test_no_filters(self): - with self.assertRaises(ValueError): - self._callFUT() - - def test_single_filter(self): - from gcloud_bigtable.row import RowFilter - - versions = 1337 - result = self._callFUT(versions=versions) - self.assertTrue(isinstance(result, RowFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(result.cells_per_column_limit_filter, versions) - - def test_existing_filters(self): - from gcloud_bigtable.row import RowFilter - - filters = [] - versions = 1337 - result = self._callFUT(versions=versions, filters=filters) - # Make sure filters has grown. - self.assertEqual(filters, [result]) - - self.assertTrue(isinstance(result, RowFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(result.cells_per_column_limit_filter, versions) - - def _column_helper(self, num_filters, versions=None, timestamp=None): - from gcloud_bigtable.row import RowFilter - from gcloud_bigtable.row import RowFilterChain - - col_fam = 'cf1' - qual = 'qual' - column = col_fam + ':' + qual - result = self._callFUT(column, versions=versions, timestamp=timestamp) - self.assertTrue(isinstance(result, RowFilterChain)) - - self.assertEqual(len(result.filters), num_filters) - fam_filter = result.filters[0] - qual_filter = result.filters[1] - self.assertTrue(isinstance(fam_filter, RowFilter)) - self.assertTrue(isinstance(qual_filter, RowFilter)) - - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(fam_filter.family_name_regex_filter, col_fam) - self.assertEqual(qual_filter.column_qualifier_regex_filter, qual) - - return result - - def test_column_only(self): - self._column_helper(num_filters=2) - - def test_with_versions(self): - from gcloud_bigtable.row import RowFilter - - versions = 11 - result = self._column_helper(num_filters=3, versions=versions) - - version_filter = result.filters[2] - self.assertTrue(isinstance(version_filter, RowFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - self.assertEqual(version_filter.cells_per_column_limit_filter, - versions) - - def test_with_timestamp(self): - from gcloud_bigtable._helpers import _microseconds_to_timestamp - from gcloud_bigtable.row import RowFilter - from gcloud_bigtable.row import TimestampRange - - timestamp = 1441928298571 - result = self._column_helper(num_filters=3, timestamp=timestamp) - - range_filter = result.filters[2] - self.assertTrue(isinstance(range_filter, RowFilter)) - # Relies on the fact that RowFilter instances can - # only have one value set. - time_range = range_filter.timestamp_range_filter - self.assertTrue(isinstance(time_range, TimestampRange)) - self.assertEqual(time_range.start, None) - ts_dt = _microseconds_to_timestamp(1000 * timestamp) - self.assertEqual(time_range.end, ts_dt) - - def test_with_all_options(self): - versions = 11 - timestamp = 1441928298571 - self._column_helper(num_filters=4, versions=versions, - timestamp=timestamp) - - -class Test__columns_filter_helper(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _columns_filter_helper - return _columns_filter_helper(*args, **kwargs) - - def test_no_columns(self): - columns = [] - with self.assertRaises(ValueError): - self._callFUT(columns) - - def test_single_column(self): - from gcloud_bigtable.row import RowFilter - - col_fam = 'cf1' - columns = [col_fam] - result = self._callFUT(columns) - expected_result = RowFilter(family_name_regex_filter=col_fam) - self.assertEqual(result, expected_result) - - def test_column_and_column_familieis(self): - from gcloud_bigtable.row import RowFilter - from gcloud_bigtable.row import RowFilterChain - from gcloud_bigtable.row import RowFilterUnion - - col_fam1 = 'cf1' - col_fam2 = 'cf2' - col_qual2 = 'qual2' - columns = [col_fam1, col_fam2 + ':' + col_qual2] - result = self._callFUT(columns) - - self.assertTrue(isinstance(result, RowFilterUnion)) - self.assertEqual(len(result.filters), 2) - filter1 = result.filters[0] - filter2 = result.filters[1] - - self.assertTrue(isinstance(filter1, RowFilter)) - self.assertEqual(filter1.family_name_regex_filter, col_fam1) - - self.assertTrue(isinstance(filter2, RowFilterChain)) - filter2a, filter2b = filter2.filters - self.assertTrue(isinstance(filter2a, RowFilter)) - self.assertEqual(filter2a.family_name_regex_filter, col_fam2) - self.assertTrue(isinstance(filter2b, RowFilter)) - self.assertEqual(filter2b.column_qualifier_regex_filter, col_qual2) - - -class Test__row_keys_filter_helper(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _row_keys_filter_helper - return _row_keys_filter_helper(*args, **kwargs) - - def test_no_rows(self): - row_keys = [] - with self.assertRaises(ValueError): - self._callFUT(row_keys) - - def test_single_row(self): - from gcloud_bigtable.row import RowFilter - - row_key = b'row-key' - row_keys = [row_key] - result = self._callFUT(row_keys) - expected_result = RowFilter(row_key_regex_filter=row_key) - self.assertEqual(result, expected_result) - - def test_many_rows(self): - from gcloud_bigtable.row import RowFilter - from gcloud_bigtable.row import RowFilterUnion - - row_key1 = b'row-key1' - row_key2 = b'row-key2' - row_key3 = b'row-key3' - row_keys = [row_key1, row_key2, row_key3] - result = self._callFUT(row_keys) - - filter1 = RowFilter(row_key_regex_filter=row_key1) - filter2 = RowFilter(row_key_regex_filter=row_key2) - filter3 = RowFilter(row_key_regex_filter=row_key3) - expected_result = RowFilterUnion(filters=[filter1, filter2, filter3]) - self.assertEqual(result, expected_result) - - -class Test__string_successor(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable.happybase.table import _string_successor - return _string_successor(*args, **kwargs) - - def test_with_alphanumeric(self): - self.assertEqual(self._callFUT(b'boa'), b'bob') - self.assertEqual(self._callFUT(b'abc1'), b'abc2') - - def test_with_last_byte(self): - self.assertEqual(self._callFUT(b'boa\xff'), b'bob') - - def test_with_empty_string(self): - self.assertEqual(self._callFUT(b''), b'') - - def test_with_all_last_bytes(self): - self.assertEqual(self._callFUT(b'\xff\xff\xff'), b'') - - def test_with_unicode_input(self): - self.assertEqual(self._callFUT(u'boa'), b'bob') - - -class TestTable(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.happybase.table import Table - return Table - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - cluster = object() - connection = _Connection(cluster) - tables_constructed = [] - - def make_low_level_table(*args, **kwargs): - result = _MockLowLevelTable(*args, **kwargs) - tables_constructed.append(result) - return result - - with _Monkey(MUT, _LowLevelTable=make_low_level_table): - table = self._makeOne(name, connection) - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - - table_instance, = tables_constructed - self.assertEqual(table._low_level_table, table_instance) - self.assertEqual(table_instance.args, (name, cluster)) - self.assertEqual(table_instance.kwargs, {}) - - def test_constructor_null_connection(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - self.assertEqual(table.name, name) - self.assertEqual(table.connection, connection) - self.assertEqual(table._low_level_table, None) - - def test___repr__(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - self.assertEqual(repr(table), '') - - def test_families(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - - # Mock the column families to be returned. - col_fam_name = 'fam' - gc_rule = object() - col_fam = _MockLowLevelColumnFamily(col_fam_name, gc_rule=gc_rule) - col_fams = {col_fam_name: col_fam} - table._low_level_table.column_families = col_fams - - to_dict_result = object() - mock_gc_rule_to_dict = _MockCalled(to_dict_result) - with _Monkey(MUT, _gc_rule_to_dict=mock_gc_rule_to_dict): - result = table.families() - - self.assertEqual(result, {col_fam_name: to_dict_result}) - self.assertEqual(table._low_level_table.list_column_families_calls, 1) - - # Check the input to our mock. - mock_gc_rule_to_dict.check_called(self, [(gc_rule,)]) - - def test_regions(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - with self.assertRaises(NotImplementedError): - table.regions() - - def test_row_empty_row(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - table._low_level_table.read_row_result = None - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - row_key = 'row-key' - timestamp = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper): - result = table.row(row_key, timestamp=timestamp) - - # read_row_result == None --> No results. - self.assertEqual(result, {}) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - expected_kwargs = { - 'filters': [], - 'versions': 1, - 'timestamp': timestamp, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_row_with_columns(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - table._low_level_table.read_row_result = None - - fake_col_filter = object() - mock_columns_filter_helper = _MockCalled(fake_col_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - row_key = 'row-key' - columns = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _columns_filter_helper=mock_columns_filter_helper): - result = table.row(row_key, columns=columns) - - # read_row_result == None --> No results. - self.assertEqual(result, {}) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - mock_columns_filter_helper.check_called(self, [(columns,)]) - expected_kwargs = { - 'filters': [fake_col_filter], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_row_with_results(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.row_data import PartialRowData - - row_key = 'row-key' - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - partial_row = PartialRowData(row_key) - table._low_level_table.read_row_result = partial_row - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - fake_pair = object() - mock_cells_to_pairs = _MockCalled([fake_pair]) - - col_fam = u'cf1' - qual = b'qual' - fake_cells = object() - partial_row._cells = {col_fam: {qual: fake_cells}} - include_timestamp = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _cells_to_pairs=mock_cells_to_pairs): - result = table.row(row_key, include_timestamp=include_timestamp) - - # The results come from _cells_to_pairs. - expected_result = {col_fam.encode('ascii') + b':' + qual: fake_pair} - self.assertEqual(result, expected_result) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - expected_kwargs = { - 'filters': [], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - to_pairs_kwargs = {'include_timestamp': include_timestamp} - mock_cells_to_pairs.check_called( - self, [(fake_cells,)], [to_pairs_kwargs]) - - def test_rows_empty_row(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - result = table.rows([]) - self.assertEqual(result, []) - - def test_rows_with_columns(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - rr_result = _MockPartialRowsData() - table._low_level_table.read_rows_result = rr_result - self.assertEqual(rr_result.consume_all_calls, 0) - - fake_col_filter = object() - mock_columns_filter_helper = _MockCalled(fake_col_filter) - fake_rows_filter = object() - mock_row_keys_filter_helper = _MockCalled(fake_rows_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - rows = ['row-key'] - columns = object() - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _row_keys_filter_helper=mock_row_keys_filter_helper, - _columns_filter_helper=mock_columns_filter_helper): - result = table.rows(rows, columns=columns) - - # read_rows_result == Empty PartialRowsData --> No results. - self.assertEqual(result, []) - - read_rows_args = () - read_rows_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_rows_calls, [ - (read_rows_args, read_rows_kwargs), - ]) - self.assertEqual(rr_result.consume_all_calls, 1) - - mock_columns_filter_helper.check_called(self, [(columns,)]) - mock_row_keys_filter_helper.check_called(self, [(rows,)]) - expected_kwargs = { - 'filters': [fake_col_filter, fake_rows_filter], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_rows_with_results(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.row_data import PartialRowData - - row_key1 = 'row-key1' - row_key2 = 'row-key2' - rows = [row_key1, row_key2] - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - - row1 = PartialRowData(row_key1) - # Return row1 but not row2 - rr_result = _MockPartialRowsData(rows={row_key1: row1}) - table._low_level_table.read_rows_result = rr_result - self.assertEqual(rr_result.consume_all_calls, 0) - - fake_rows_filter = object() - mock_row_keys_filter_helper = _MockCalled(fake_rows_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - fake_pair = object() - mock_cells_to_pairs = _MockCalled([fake_pair]) - - col_fam = u'cf1' - qual = b'qual' - fake_cells = object() - row1._cells = {col_fam: {qual: fake_cells}} - include_timestamp = object() - with _Monkey(MUT, _row_keys_filter_helper=mock_row_keys_filter_helper, - _filter_chain_helper=mock_filter_chain_helper, - _cells_to_pairs=mock_cells_to_pairs): - result = table.rows(rows, include_timestamp=include_timestamp) - - # read_rows_result == PartialRowsData with row_key1 - expected_result = {col_fam.encode('ascii') + b':' + qual: fake_pair} - self.assertEqual(result, [(row_key1, expected_result)]) - - read_rows_args = () - read_rows_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_rows_calls, [ - (read_rows_args, read_rows_kwargs), - ]) - self.assertEqual(rr_result.consume_all_calls, 1) - - mock_row_keys_filter_helper.check_called(self, [(rows,)]) - expected_kwargs = { - 'filters': [fake_rows_filter], - 'versions': 1, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - to_pairs_kwargs = {'include_timestamp': include_timestamp} - mock_cells_to_pairs.check_called( - self, [(fake_cells,)], [to_pairs_kwargs]) - - def test_cells_empty_row(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - table._low_level_table.read_row_result = None - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - row_key = 'row-key' - column = 'fam:col1' - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper): - result = table.cells(row_key, column) - - # read_row_result == None --> No results. - self.assertEqual(result, []) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - expected_kwargs = { - 'column': column, - 'versions': None, - 'timestamp': None, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_cells_with_results(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.row_data import PartialRowData - - row_key = 'row-key' - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - partial_row = PartialRowData(row_key) - table._low_level_table.read_row_result = partial_row - - # These are all passed to mocks. - versions = object() - timestamp = object() - include_timestamp = object() - - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - fake_result = object() - mock_cells_to_pairs = _MockCalled(fake_result) - - col_fam = 'cf1' - qual = 'qual' - fake_cells = object() - partial_row._cells = {col_fam: {qual: fake_cells}} - column = col_fam + ':' + qual - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _cells_to_pairs=mock_cells_to_pairs): - result = table.cells(row_key, column, versions=versions, - timestamp=timestamp, - include_timestamp=include_timestamp) - - self.assertEqual(result, fake_result) - - read_row_args = (row_key,) - read_row_kwargs = {'filter_': fake_filter} - self.assertEqual(table._low_level_table.read_row_calls, [ - (read_row_args, read_row_kwargs), - ]) - - filter_kwargs = { - 'column': column, - 'versions': versions, - 'timestamp': timestamp, - } - mock_filter_chain_helper.check_called(self, [()], [filter_kwargs]) - to_pairs_kwargs = {'include_timestamp': include_timestamp} - mock_cells_to_pairs.check_called( - self, [(fake_cells,)], [to_pairs_kwargs]) - - def test_scan_with_batch_size(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(batch_size=object())) - - def test_scan_with_scan_batching(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(scan_batching=object())) - - def test_scan_with_sorted_columns(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(sorted_columns=object())) - - def test_scan_with_invalid_limit(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(limit=-10)) - - def test_scan_with_row_prefix_and_row_start(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(ValueError): - list(table.scan(row_prefix='a', row_stop='abc')) - - def test_scan_with_string_filter(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - with self.assertRaises(TypeError): - list(table.scan(filter='some-string')) - - def _scan_test_helper(self, row_start=None, row_stop=None, row_prefix=None, - columns=None, filter_=None, timestamp=None, - include_timestamp=False, limit=None, rr_result=None, - expected_result=None): - import types - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - table._low_level_table = _MockLowLevelTable() - rr_result = rr_result or _MockPartialRowsData() - table._low_level_table.read_rows_result = rr_result - self.assertEqual(rr_result.consume_next_calls, 0) - - fake_col_filter = object() - mock_columns_filter_helper = _MockCalled(fake_col_filter) - fake_filter = object() - mock_filter_chain_helper = _MockCalled(fake_filter) - - with _Monkey(MUT, _filter_chain_helper=mock_filter_chain_helper, - _columns_filter_helper=mock_columns_filter_helper): - result = table.scan(row_start=row_start, row_stop=row_stop, - row_prefix=row_prefix, columns=columns, - filter=filter_, timestamp=timestamp, - include_timestamp=include_timestamp, - limit=limit) - self.assertTrue(isinstance(result, types.GeneratorType)) - # Need to consume the result while the monkey patch is applied. - # read_rows_result == Empty PartialRowsData --> No results. - expected_result = expected_result or [] - self.assertEqual(list(result), expected_result) - - read_rows_args = () - if row_prefix: - row_start = row_prefix - row_stop = MUT._string_successor(row_prefix) - read_rows_kwargs = { - 'end_key': row_stop, - 'filter_': fake_filter, - 'limit': limit, - 'start_key': row_start, - } - self.assertEqual(table._low_level_table.read_rows_calls, [ - (read_rows_args, read_rows_kwargs), - ]) - self.assertEqual(rr_result.consume_next_calls, - rr_result.iterations + 1) - - if columns is not None: - mock_columns_filter_helper.check_called(self, [(columns,)]) - else: - mock_columns_filter_helper.check_called(self, []) - - filters = [] - if filter_ is not None: - filters.append(filter_) - if columns: - filters.append(fake_col_filter) - expected_kwargs = { - 'filters': filters, - 'versions': 1, - 'timestamp': timestamp, - } - mock_filter_chain_helper.check_called(self, [()], [expected_kwargs]) - - def test_scan_with_columns(self): - columns = object() - self._scan_test_helper(columns=columns) - - def test_scan_with_row_start_and_stop(self): - row_start = 'bar' - row_stop = 'foo' - self._scan_test_helper(row_start=row_start, row_stop=row_stop) - - def test_scan_with_row_prefix(self): - row_prefix = 'row-prefi' - self._scan_test_helper(row_prefix=row_prefix) - - def test_scan_with_filter(self): - mock_filter = object() - self._scan_test_helper(filter_=mock_filter) - - def test_scan_with_no_results(self): - limit = 1337 - timestamp = object() - self._scan_test_helper(timestamp=timestamp, limit=limit) - - def test_scan_with_results(self): - from gcloud_bigtable.row_data import PartialRowData - - row_key1 = 'row-key1' - row1 = PartialRowData(row_key1) - rr_result = _MockPartialRowsData(rows={row_key1: row1}, iterations=1) - - include_timestamp = object() - expected_result = [(row_key1, {})] - self._scan_test_helper(include_timestamp=include_timestamp, - rr_result=rr_result, - expected_result=expected_result) - - def test_put(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.happybase.table import _WAL_SENTINEL - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - batches_created = [] - - def make_batch(*args, **kwargs): - result = _MockBatch(*args, **kwargs) - batches_created.append(result) - return result - - row = 'row-key' - data = {'fam:col': 'foo'} - timestamp = None - with _Monkey(MUT, Batch=make_batch): - result = table.put(row, data, timestamp=timestamp) - - # There is no return value. - self.assertEqual(result, None) - - # Check how the batch was created and used. - batch, = batches_created - self.assertTrue(isinstance(batch, _MockBatch)) - self.assertEqual(batch.args, (table,)) - expected_kwargs = { - 'timestamp': timestamp, - 'batch_size': None, - 'transaction': False, - 'wal': _WAL_SENTINEL, - } - self.assertEqual(batch.kwargs, expected_kwargs) - # Make sure it was a successful context manager - self.assertEqual(batch.exit_vals, [(None, None, None)]) - self.assertEqual(batch.put_args, [(row, data)]) - self.assertEqual(batch.delete_args, []) - - def test_delete(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - from gcloud_bigtable.happybase.table import _WAL_SENTINEL - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - batches_created = [] - - def make_batch(*args, **kwargs): - result = _MockBatch(*args, **kwargs) - batches_created.append(result) - return result - - row = 'row-key' - columns = ['fam:col1', 'fam:col2'] - timestamp = None - with _Monkey(MUT, Batch=make_batch): - result = table.delete(row, columns=columns, timestamp=timestamp) - - # There is no return value. - self.assertEqual(result, None) - - # Check how the batch was created and used. - batch, = batches_created - self.assertTrue(isinstance(batch, _MockBatch)) - self.assertEqual(batch.args, (table,)) - expected_kwargs = { - 'timestamp': timestamp, - 'batch_size': None, - 'transaction': False, - 'wal': _WAL_SENTINEL, - } - self.assertEqual(batch.kwargs, expected_kwargs) - # Make sure it was a successful context manager - self.assertEqual(batch.exit_vals, [(None, None, None)]) - self.assertEqual(batch.put_args, []) - self.assertEqual(batch.delete_args, [(row, columns)]) - - def test_batch(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.happybase import table as MUT - - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - timestamp = object() - batch_size = 42 - transaction = False # Must be False when batch_size is non-null - wal = object() - - with _Monkey(MUT, Batch=_MockBatch): - result = table.batch(timestamp=timestamp, batch_size=batch_size, - transaction=transaction, wal=wal) - - self.assertTrue(isinstance(result, _MockBatch)) - self.assertEqual(result.args, (table,)) - expected_kwargs = { - 'timestamp': timestamp, - 'batch_size': batch_size, - 'transaction': transaction, - 'wal': wal, - } - self.assertEqual(result.kwargs, expected_kwargs) - - def test_counter_get(self): - klass = self._getTargetClass() - counter_value = 1337 - - class TableWithInc(klass): - - incremented = [] - value = counter_value - - def counter_inc(self, row, column, value=1): - self.incremented.append((row, column, value)) - self.value += value - return self.value - - name = 'table-name' - connection = None - table = TableWithInc(name, connection) - - row = 'row-key' - column = 'fam:col1' - self.assertEqual(TableWithInc.incremented, []) - result = table.counter_get(row, column) - self.assertEqual(result, counter_value) - self.assertEqual(TableWithInc.incremented, [(row, column, 0)]) - - def test_counter_set(self): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - - row = 'row-key' - column = 'fam:col1' - value = 42 - with self.assertRaises(NotImplementedError): - table.counter_set(row, column, value=value) - - def test_counter_dec(self): - klass = self._getTargetClass() - counter_value = 42 - - class TableWithInc(klass): - - incremented = [] - value = counter_value - - def counter_inc(self, row, column, value=1): - self.incremented.append((row, column, value)) - self.value += value - return self.value - - name = 'table-name' - connection = None - table = TableWithInc(name, connection) - - row = 'row-key' - column = 'fam:col1' - dec_value = 987 - self.assertEqual(TableWithInc.incremented, []) - result = table.counter_dec(row, column, value=dec_value) - self.assertEqual(result, counter_value - dec_value) - self.assertEqual(TableWithInc.incremented, [(row, column, -dec_value)]) - - def _counter_inc_helper(self, row, column, value, commit_result): - name = 'table-name' - connection = None - table = self._makeOne(name, connection) - # Mock the return values. - table._low_level_table = _MockLowLevelTable() - table._low_level_table.row_values[row] = _MockLowLevelRow( - row, commit_result=commit_result) - - result = table.counter_inc(row, column, value=value) - - incremented_value = value + _MockLowLevelRow.COUNTER_DEFAULT - self.assertEqual(result, incremented_value) - - # Check the row values returned. - row_obj = table._low_level_table.row_values[row] - self.assertEqual(row_obj.counts, - {tuple(column.split(':')): incremented_value}) - - def test_counter_inc(self): - import struct - - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - packed_value = struct.pack('>q', value) - fake_timestamp = None - commit_result = { - col_fam: { - col_qual: [(packed_value, fake_timestamp)], - } - } - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_bad_result(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - commit_result = None - with self.assertRaises(TypeError): - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_result_key_error(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - commit_result = {} - with self.assertRaises(KeyError): - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_result_nested_key_error(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - commit_result = {col_fam: {}} - with self.assertRaises(KeyError): - self._counter_inc_helper(row, column, value, commit_result) - - def test_counter_inc_result_non_unique_cell(self): - row = 'row-key' - col_fam = 'fam' - col_qual = 'col1' - column = col_fam + ':' + col_qual - value = 42 - fake_timestamp = None - packed_value = None - commit_result = { - col_fam: { - col_qual: [ - (packed_value, fake_timestamp), - (packed_value, fake_timestamp), - ], - } - } - with self.assertRaises(ValueError): - self._counter_inc_helper(row, column, value, commit_result) - - -class _MockLowLevelTable(object): - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.list_column_families_calls = 0 - self.column_families = {} - self.row_values = {} - self.read_row_calls = [] - self.read_row_result = None - self.read_rows_calls = [] - self.read_rows_result = None - - def list_column_families(self): - self.list_column_families_calls += 1 - return self.column_families - - def row(self, row_key): - return self.row_values[row_key] - - def read_row(self, *args, **kwargs): - self.read_row_calls.append((args, kwargs)) - return self.read_row_result - - def read_rows(self, *args, **kwargs): - self.read_rows_calls.append((args, kwargs)) - return self.read_rows_result - - -class _MockLowLevelColumnFamily(object): - - def __init__(self, column_family_id, gc_rule=None): - self.column_family_id = column_family_id - self.gc_rule = gc_rule - - -class _Connection(object): - - def __init__(self, cluster): - self._cluster = cluster - - -class _MockBatch(object): - - def __init__(self, *args, **kwargs): - self.args = args - self.kwargs = kwargs - self.exit_vals = [] - self.put_args = [] - self.delete_args = [] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_value, traceback): - self.exit_vals.append((exc_type, exc_value, traceback)) - - def put(self, *args): - self.put_args.append(args) - - def delete(self, *args): - self.delete_args.append(args) - - -class _MockLowLevelRow(object): - - COUNTER_DEFAULT = 0 - - def __init__(self, row_key, commit_result=None): - self.row_key = row_key - self.counts = {} - self.commit_result = commit_result - - def increment_cell_value(self, column_family_id, column, int_value): - count = self.counts.setdefault((column_family_id, column), - self.COUNTER_DEFAULT) - self.counts[(column_family_id, column)] = count + int_value - - def commit_modifications(self): - return self.commit_result - - -class _MockPartialRowsData(object): - - def __init__(self, rows=None, iterations=0): - self.rows = rows or {} - self.consume_all_calls = 0 - self.consume_next_calls = 0 - self.iterations = iterations - - def consume_all(self): - self.consume_all_calls += 1 - - def consume_next(self): - self.consume_next_calls += 1 - if self.consume_next_calls > self.iterations: - raise StopIteration diff --git a/gcloud_bigtable/row.py b/gcloud_bigtable/row.py deleted file mode 100644 index 5d26c49..0000000 --- a/gcloud_bigtable/row.py +++ /dev/null @@ -1,1210 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Row.""" - - -import six -import struct - -from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._helpers import _parse_family_pb -from gcloud_bigtable._helpers import _timestamp_to_microseconds -from gcloud_bigtable._helpers import _to_bytes - - -_MAX_MUTATIONS = 100000 -_PACK_I64 = struct.Struct('>q').pack - - -class Row(object): - """Representation of a Google Cloud Bigtable Row. - - .. note:: - - A :class:`Row` accumulates mutations locally via the :meth:`set_cell`, - :meth:`delete`, :meth:`delete_cell` and :meth:`delete_cells` methods. - To actually send these mutations to the Google Cloud Bigtable API, you - must call :meth:`commit`. If a ``filter_`` is set on the :class:`Row`, - the mutations must have an associated state: :data:`True` or - :data:`False`. The mutations will be applied conditionally, based on - whether the filter matches any cells in the :class:`Row` or not. - - :type row_key: bytes - :param row_key: The key for the current row. - - :type table: :class:`Table ` - :param table: The table that owns the row. - - :type filter_: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` - :param filter_: (Optional) Filter to be used for conditional mutations. - If a filter is set, then the :class:`Row` will accumulate - mutations for either a :data:`True` or :data:`False` state. - When :meth:`commit`-ed, the mutations for the :data:`True` - state will be applied if the filter matches any cells in - the row, otherwise the :data:`False` state will be. - """ - - ALL_COLUMNS = object() - """Sentinel value used to indicate all columns in a column family.""" - - def __init__(self, row_key, table, filter_=None): - self._row_key = _to_bytes(row_key) - self._table = table - self._filter = filter_ - self._rule_pb_list = [] - if self._filter is None: - self._pb_mutations = [] - self._true_pb_mutations = None - self._false_pb_mutations = None - else: - self._pb_mutations = None - self._true_pb_mutations = [] - self._false_pb_mutations = [] - - @property - def table(self): - """Getter for row's table. - - :rtype: :class:`Table ` - :returns: The table stored on the row. - """ - return self._table - - @property - def row_key(self): - """Getter for row's key. - - :rtype: bytes - :returns: The key for the row. - """ - return self._row_key - - @property - def filter(self): - """Getter for row's filter. - - :rtype: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion`, :class:`ConditionalRowFilter` or - :data:`NoneType ` - :returns: The filter for the row. - """ - return self._filter - - @property - def client(self): - """Getter for row's client. - - :rtype: :class:`.client.Client` - :returns: The client that owns this row. - """ - return self.table.client - - @property - def timeout_seconds(self): - """Getter for row's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default. - """ - return self.table.timeout_seconds - - def _get_mutations(self, state=None): - """Gets the list of mutations for a given state. - - If the state is :data`None` but there is a filter set, then we've - reached an invalid state. Similarly if no filter is set but the - state is not :data:`None`. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - - :rtype: list - :returns: The list to add new mutations to (for the current state). - :raises: :class:`ValueError ` - """ - if state is None: - if self.filter is not None: - raise ValueError('A filter is set on the current row, but no ' - 'state given for the mutation') - return self._pb_mutations - else: - if self.filter is None: - raise ValueError('No filter was set on the current row, but a ' - 'state was given for the mutation') - if state: - return self._true_pb_mutations - else: - return self._false_pb_mutations - - def set_cell(self, column_family_id, column, value, timestamp=None, - state=None): - """Sets a value in this row. - - The cell is determined by the ``row_key`` of the :class:`Row` and the - ``column``. The ``column`` must be in an existing - :class:`.column_family.ColumnFamily` (as determined by - ``column_family_id``). - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column. - Must be of the form - ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family where the cell - is located. - - :type value: bytes or :class:`int` - :param value: The value to set in the cell. If an integer is used, - will be interpreted as a 64-bit big-endian signed - integer (8 bytes). - - :type timestamp: :class:`datetime.datetime` - :param timestamp: (Optional) The timestamp of the operation. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - column = _to_bytes(column) - if isinstance(value, six.integer_types): - value = _PACK_I64(value) - value = _to_bytes(value) - if timestamp is None: - # Use -1 for current Bigtable server time. - timestamp_micros = -1 - else: - timestamp_micros = _timestamp_to_microseconds(timestamp) - - mutation_val = data_pb2.Mutation.SetCell( - family_name=column_family_id, - column_qualifier=column, - timestamp_micros=timestamp_micros, - value=value, - ) - mutation_pb = data_pb2.Mutation(set_cell=mutation_val) - self._get_mutations(state).append(mutation_pb) - - def append_cell_value(self, column_family_id, column, value): - """Appends a value to an existing cell. - - .. note:: - - This method adds a read-modify rule protobuf to the accumulated - read-modify rules on this :class:`Row`, but does not make an API - request. To actually send an API request (with the rules) to the - Google Cloud Bigtable API, call :meth:`commit_modifications`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column. - Must be of the form - ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family where the cell - is located. - - :type value: bytes - :param value: The value to append to the existing value in the cell. If - the targeted cell is unset, it will be treated as - containing the empty string. - """ - column = _to_bytes(column) - value = _to_bytes(value) - rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, - column_qualifier=column, - append_value=value) - self._rule_pb_list.append(rule_pb) - - def increment_cell_value(self, column_family_id, column, int_value): - """Increments a value in an existing cell. - - Assumes the value in the cell is stored as a 64 bit integer - serialized to bytes. - - .. note:: - - This method adds a read-modify rule protobuf to the accumulated - read-modify rules on this :class:`Row`, but does not make an API - request. To actually send an API request (with the rules) to the - Google Cloud Bigtable API, call :meth:`commit_modifications`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column. - Must be of the form - ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family where the cell - is located. - - :type int_value: int - :param int_value: The value to increment the existing value in the cell - by. If the targeted cell is unset, it will be treated - as containing a zero. Otherwise, the targeted cell - must contain an 8-byte value (interpreted as a 64-bit - big-endian signed integer), or the entire request - will fail. - """ - column = _to_bytes(column) - rule_pb = data_pb2.ReadModifyWriteRule(family_name=column_family_id, - column_qualifier=column, - increment_amount=int_value) - self._rule_pb_list.append(rule_pb) - - def delete(self, state=None): - """Deletes this row from the table. - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - mutation_val = data_pb2.Mutation.DeleteFromRow() - mutation_pb = data_pb2.Mutation(delete_from_row=mutation_val) - self._get_mutations(state).append(mutation_pb) - - def delete_cell(self, column_family_id, column, time_range=None, - state=None): - """Deletes cell in this row. - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column - or columns with cells being deleted. Must be - of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type column: bytes - :param column: The column within the column family that will have a - cell deleted. - - :type time_range: :class:`TimestampRange` - :param time_range: (Optional) The range of time within which cells - should be deleted. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - self.delete_cells(column_family_id, [column], time_range=time_range, - state=state) - - def delete_cells(self, column_family_id, columns, time_range=None, - state=None): - """Deletes cells in this row. - - .. note:: - - This method adds a mutation to the accumulated mutations on this - :class:`Row`, but does not make an API request. To actually - send an API request (with the mutations) to the Google Cloud - Bigtable API, call :meth:`commit`. - - :type column_family_id: str - :param column_family_id: The column family that contains the column - or columns with cells being deleted. Must be - of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type columns: :class:`list` of :class:`str` / - :func:`unicode `, or :class:`object` - :param columns: The columns within the column family that will have - cells deleted. If :attr:`Row.ALL_COLUMNS` is used then - the entire column family will be deleted from the row. - - :type time_range: :class:`TimestampRange` - :param time_range: (Optional) The range of time within which cells - should be deleted. - - :type state: bool - :param state: (Optional) The state that the mutation should be - applied in. Unset if the mutation is not conditional, - otherwise :data:`True` or :data:`False`. - """ - mutations_list = self._get_mutations(state) - if columns is self.ALL_COLUMNS: - mutation_val = data_pb2.Mutation.DeleteFromFamily( - family_name=column_family_id, - ) - mutation_pb = data_pb2.Mutation(delete_from_family=mutation_val) - mutations_list.append(mutation_pb) - else: - delete_kwargs = {} - if time_range is not None: - delete_kwargs['time_range'] = time_range.to_pb() - - to_append = [] - for column in columns: - column = _to_bytes(column) - # time_range will never change if present, but the rest of - # delete_kwargs will - delete_kwargs.update( - family_name=column_family_id, - column_qualifier=column, - ) - mutation_val = data_pb2.Mutation.DeleteFromColumn( - **delete_kwargs) - mutation_pb = data_pb2.Mutation( - delete_from_column=mutation_val) - to_append.append(mutation_pb) - - # We don't add the mutations until all columns have been - # processed without error. - mutations_list.extend(to_append) - - def _commit_mutate(self, timeout_seconds=None): - """Makes a ``MutateRow`` API request. - - Assumes no filter is set on the :class:`Row` and is meant to be called - by :meth:`commit`. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - - :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. - """ - mutations_list = self._get_mutations(None) - num_mutations = len(mutations_list) - if num_mutations == 0: - return - if num_mutations > _MAX_MUTATIONS: - raise ValueError('%d total mutations exceed the maximum allowable ' - '%d.' % (num_mutations, _MAX_MUTATIONS)) - request_pb = messages_pb2.MutateRowRequest( - table_name=self.table.name, - row_key=self.row_key, - mutations=mutations_list, - ) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._data_stub.MutateRow.async(request_pb, - timeout_seconds) - # We expect a `._generated.empty_pb2.Empty`. - response.result() - - def _commit_check_and_mutate(self, timeout_seconds=None): - """Makes a ``CheckAndMutateRow`` API request. - - Assumes a filter is set on the :class:`Row` and is meant to be called - by :meth:`commit`. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - - :rtype: bool - :returns: Flag indicating if the filter was matched (which also - indicates which set of mutations were applied by the server). - :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. - """ - true_mutations = self._get_mutations(True) - false_mutations = self._get_mutations(False) - num_true_mutations = len(true_mutations) - num_false_mutations = len(false_mutations) - if num_true_mutations == 0 and num_false_mutations == 0: - return - if (num_true_mutations > _MAX_MUTATIONS or - num_false_mutations > _MAX_MUTATIONS): - raise ValueError( - 'Exceed the maximum allowable mutations (%d). Had %s true ' - 'mutations and %d false mutations.' % ( - _MAX_MUTATIONS, num_true_mutations, num_false_mutations)) - - request_pb = messages_pb2.CheckAndMutateRowRequest( - table_name=self.table.name, - row_key=self.row_key, - predicate_filter=self.filter.to_pb(), - true_mutations=true_mutations, - false_mutations=false_mutations, - ) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._data_stub.CheckAndMutateRow.async( - request_pb, timeout_seconds) - # We expect a `.messages_pb2.CheckAndMutateRowResponse` - check_and_mutate_row_response = response.result() - return check_and_mutate_row_response.predicate_matched - - def clear_mutations(self): - """Removes all currently accumulated mutations on the current row.""" - if self.filter is None: - self._pb_mutations[:] = [] - else: - self._true_pb_mutations[:] = [] - self._false_pb_mutations[:] = [] - - def commit(self, timeout_seconds=None): - """Makes a ``MutateRow`` or ``CheckAndMutateRow`` API request. - - If no mutations have been created in the row, no request is made. - - Mutations are applied atomically and in order, meaning that earlier - mutations can be masked / negated by later ones. Cells already present - in the row are left unchanged unless explicitly changed by a mutation. - - After committing the accumulated mutations, resets the local - mutations to an empty list. - - In the case that a filter is set on the :class:`Row`, the mutations - will be applied conditionally, based on whether the filter matches - any cells in the :class:`Row` or not. (Each method which adds a - mutation has a ``state`` parameter for this purpose.) - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - - :rtype: :class:`bool` or :data:`NoneType ` - :returns: :data:`None` if there is no filter, otherwise a flag - indicating if the filter was matched (which also - indicates which set of mutations were applied by the server). - :raises: :class:`ValueError ` if the number of - mutations exceeds the ``_MAX_MUTATIONS``. - """ - if self.filter is None: - result = self._commit_mutate(timeout_seconds=timeout_seconds) - else: - result = self._commit_check_and_mutate( - timeout_seconds=timeout_seconds) - - # Reset mutations after commit-ing request. - self.clear_mutations() - - return result - - def clear_modification_rules(self): - """Removes all currently accumulated modifications on current row.""" - self._rule_pb_list[:] = [] - - def commit_modifications(self, timeout_seconds=None): - """Makes a ``ReadModifyWriteRow`` API request. - - This commits modifications made by :meth:`append_cell_value` and - :meth:`increment_cell_value`. If no modifications were made, makes - no API request and just returns ``{}``. - - Modifies a row atomically, reading the latest existing timestamp/value - from the specified columns and writing a new value by appending / - incrementing. The new cell created uses either the current server time - or the highest timestamp of a cell in that column (if it exceeds the - server time). - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on row. - - :rtype: dict - :returns: The new contents of all modified cells. Returned as a - dictionary of column families, each of which holds a - dictionary of columns. Each column contains a list of cells - modified. Each cell is represented with a two-tuple with the - value (in bytes) and the timestamp for the cell. For example: - - .. code:: python - - { - u'col-fam-id': { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - }, - u'col-fam-id2': { - b'col-name3-but-other-fam': [ - (b'foo', datetime.datetime(...)), - ], - }, - } - """ - if len(self._rule_pb_list) == 0: - return {} - request_pb = messages_pb2.ReadModifyWriteRowRequest( - table_name=self.table.name, - row_key=self.row_key, - rules=self._rule_pb_list, - ) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._data_stub.ReadModifyWriteRow.async( - request_pb, timeout_seconds) - # We expect a `.data_pb2.Row` - row_response = response.result() - - # Reset modifications after commit-ing request. - self.clear_modification_rules() - - # NOTE: We expect row_response.key == self.row_key but don't check. - return _parse_rmw_row_response(row_response) - - -# NOTE: For developers, this class may seem to be a bit verbose, i.e. -# a list of property names and **kwargs may do the trick better -# than actually listing every single argument. However, for the sake -# of users and documentation, listing every single argument is more -# useful. -# pylint: disable=too-many-instance-attributes -# pylint: disable=too-many-arguments -# pylint: disable=too-many-branches -class RowFilter(object): - """Basic filter to apply to cells in a row. - - These values can be combined via :class:`RowFilterChain`, - :class:`RowFilterUnion` and :class:`ConditionalRowFilter`. - - The regex filters must be valid RE2 patterns. See Google's - `RE2 reference`_ for the accepted syntax. - - .. _RE2 reference: https://github.com/google/re2/wiki/Syntax - - .. note:: - - At most one of the keyword arguments can be specified at once. - - .. note:: - - For :class:`bytes` regex filters (``row_key``, ``column_qualifier`` and - ``value``), special care need be used with the expression used. Since - each of these properties can contain arbitrary bytes, the ``\\C`` - escape sequence must be used if a true wildcard is desired. The ``.`` - character will not match the new line character ``\\n``, which may be - present in a binary value. - - :type sink: bool - :param sink: ADVANCED USE ONLY. Hook for introspection into the RowFilter. - Outputs all cells directly to the output of the read rather - than to any parent filter. Cannot be used within the - ``predicate_filter``, ``true_filter``, or ``false_filter`` - of a :class:`ConditionalRowFilter`. - - :type pass_all_filter: bool - :param pass_all_filter: Matches all cells, regardless of input. - Functionally equivalent to leaving ``filter`` - unset, but included for completeness. - - :type block_all_filter: bool - :param block_all_filter: Does not match any cells, regardless of input. - Useful for temporarily disabling just part of - a filter. - - :type row_key_regex_filter: bytes - :param row_key_regex_filter: A regular expression (RE2) to match cells from - rows with row keys that satisfy this regex. - For a ``CheckAndMutateRowRequest``, this - filter is unnecessary since the row key is - already specified. - - :type row_sample_filter: float - :param row_sample_filter: Non-deterministic filter. Matches all cells from - a row with probability p, and matches no cells - from the row with probability 1-p. (Here, the - probability p is ``row_sample_filter``.) - - :type family_name_regex_filter: str - :param family_name_regex_filter: A regular expression (RE2) to match cells - from columns in a given column family. For - technical reasons, the regex must not - contain the ``':'`` character, even if it - isnot being uses as a literal. - - :type column_qualifier_regex_filter: bytes - :param column_qualifier_regex_filter: A regular expression (RE2) to match - cells from column that match this - regex (irrespective of column - family). - - :type column_range_filter: :class:`ColumnRange` - :param column_range_filter: Range of columns to limit cells to. - - :type timestamp_range_filter: :class:`TimestampRange` - :param timestamp_range_filter: Range of time that cells should match - against. - - :type value_regex_filter: bytes - :param value_regex_filter: A regular expression (RE2) to match cells with - values that match this regex. - - :type value_range_filter: :class:`CellValueRange` - :param value_range_filter: Range of cell values to filter for. - - :type cells_per_row_offset_filter: int - :param cells_per_row_offset_filter: Skips the first N cells of the row. - - :type cells_per_row_limit_filter: int - :param cells_per_row_limit_filter: Matches only the first N cells of the - row. - - :type cells_per_column_limit_filter: int - :param cells_per_column_limit_filter: Matches only the most recent N cells - within each column. This filters a - (family name, column) pair, based on - timestamps of each cell. - - :type strip_value_transformer: bool - :param strip_value_transformer: If :data:`True`, replaces each cell's value - with the empty string. As the name - indicates, this is more useful as a - transformer than a generic query / filter. - - :type apply_label_transformer: str - :param apply_label_transformer: Applies the given label to all cells in the - output row. This allows the client to - determine which results were produced from - which part of the filter. - - Values must be at most 15 characters long, - and match the pattern [a-z0-9\\-]+. - - Due to a technical limitation, it is not - currently possible to apply multiple labels - to a cell. - - :raises: :class:`TypeError ` if not exactly one - value set in the constructor. - """ - - def __init__(self, - sink=None, - pass_all_filter=None, - block_all_filter=None, - row_key_regex_filter=None, - row_sample_filter=None, - family_name_regex_filter=None, - column_qualifier_regex_filter=None, - column_range_filter=None, - timestamp_range_filter=None, - value_regex_filter=None, - value_range_filter=None, - cells_per_row_offset_filter=None, - cells_per_row_limit_filter=None, - cells_per_column_limit_filter=None, - strip_value_transformer=None, - apply_label_transformer=None): - self.sink = sink - self.pass_all_filter = pass_all_filter - self.block_all_filter = block_all_filter - self.row_key_regex_filter = row_key_regex_filter - self.row_sample_filter = row_sample_filter - self.family_name_regex_filter = family_name_regex_filter - self.column_qualifier_regex_filter = column_qualifier_regex_filter - self.column_range_filter = column_range_filter - self.timestamp_range_filter = timestamp_range_filter - self.value_regex_filter = value_regex_filter - self.value_range_filter = value_range_filter - self.cells_per_row_offset_filter = cells_per_row_offset_filter - self.cells_per_row_limit_filter = cells_per_row_limit_filter - self.cells_per_column_limit_filter = cells_per_column_limit_filter - self.strip_value_transformer = strip_value_transformer - self.apply_label_transformer = apply_label_transformer - self._check_single_value() - - def _check_single_value(self): - """Checks that exactly one value is set on the instance. - - :raises: :class:`TypeError ` if not exactly one - value set on the instance. - """ - values_set = ( - int(self.sink is not None) + - int(self.pass_all_filter is not None) + - int(self.block_all_filter is not None) + - int(self.row_key_regex_filter is not None) + - int(self.row_sample_filter is not None) + - int(self.family_name_regex_filter is not None) + - int(self.column_qualifier_regex_filter is not None) + - int(self.column_range_filter is not None) + - int(self.timestamp_range_filter is not None) + - int(self.value_regex_filter is not None) + - int(self.value_range_filter is not None) + - int(self.cells_per_row_offset_filter is not None) + - int(self.cells_per_row_limit_filter is not None) + - int(self.cells_per_column_limit_filter is not None) + - int(self.strip_value_transformer is not None) + - int(self.apply_label_transformer is not None) - ) - if values_set != 1: - raise TypeError('Exactly one value must be set in a row filter') - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return ( - other.sink == self.sink and - other.pass_all_filter == self.pass_all_filter and - other.block_all_filter == self.block_all_filter and - other.row_key_regex_filter == self.row_key_regex_filter and - other.row_sample_filter == self.row_sample_filter and - other.family_name_regex_filter == self.family_name_regex_filter and - (other.column_qualifier_regex_filter == - self.column_qualifier_regex_filter) and - other.column_range_filter == self.column_range_filter and - other.timestamp_range_filter == self.timestamp_range_filter and - other.value_regex_filter == self.value_regex_filter and - other.value_range_filter == self.value_range_filter and - (other.cells_per_row_offset_filter == - self.cells_per_row_offset_filter) and - (other.cells_per_row_limit_filter == - self.cells_per_row_limit_filter) and - (other.cells_per_column_limit_filter == - self.cells_per_column_limit_filter) and - other.strip_value_transformer == self.strip_value_transformer and - other.apply_label_transformer == self.apply_label_transformer - ) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`RowFilter` to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - self._check_single_value() - row_filter_kwargs = {} - if self.sink is not None: - row_filter_kwargs['sink'] = self.sink - if self.pass_all_filter is not None: - row_filter_kwargs['pass_all_filter'] = self.pass_all_filter - if self.block_all_filter is not None: - row_filter_kwargs['block_all_filter'] = self.block_all_filter - if self.row_key_regex_filter is not None: - row_filter_kwargs['row_key_regex_filter'] = _to_bytes( - self.row_key_regex_filter) - if self.row_sample_filter is not None: - row_filter_kwargs['row_sample_filter'] = ( - self.row_sample_filter) - if self.family_name_regex_filter is not None: - row_filter_kwargs['family_name_regex_filter'] = ( - self.family_name_regex_filter) - if self.column_qualifier_regex_filter is not None: - row_filter_kwargs['column_qualifier_regex_filter'] = _to_bytes( - self.column_qualifier_regex_filter) - if self.column_range_filter is not None: - row_filter_kwargs['column_range_filter'] = ( - self.column_range_filter.to_pb()) - if self.timestamp_range_filter is not None: - row_filter_kwargs['timestamp_range_filter'] = ( - self.timestamp_range_filter.to_pb()) - if self.value_regex_filter is not None: - row_filter_kwargs['value_regex_filter'] = _to_bytes( - self.value_regex_filter) - if self.value_range_filter is not None: - row_filter_kwargs['value_range_filter'] = ( - self.value_range_filter.to_pb()) - if self.cells_per_row_offset_filter is not None: - row_filter_kwargs['cells_per_row_offset_filter'] = ( - self.cells_per_row_offset_filter) - if self.cells_per_row_limit_filter is not None: - row_filter_kwargs['cells_per_row_limit_filter'] = ( - self.cells_per_row_limit_filter) - if self.cells_per_column_limit_filter is not None: - row_filter_kwargs['cells_per_column_limit_filter'] = ( - self.cells_per_column_limit_filter) - if self.strip_value_transformer is not None: - row_filter_kwargs['strip_value_transformer'] = ( - self.strip_value_transformer) - if self.apply_label_transformer is not None: - row_filter_kwargs['apply_label_transformer'] = ( - self.apply_label_transformer) - return data_pb2.RowFilter(**row_filter_kwargs) -# pylint: enable=too-many-instance-attributes -# pylint: enable=too-many-arguments -# pylint: enable=too-many-branches - - -class TimestampRange(object): - """Range of time with inclusive lower and exclusive upper bounds. - - :type start: :class:`datetime.datetime` - :param start: (Optional) The (inclusive) lower bound of the timestamp - range. If omitted, defaults to Unix epoch. - - :type end: :class:`datetime.datetime` - :param end: (Optional) The (exclusive) upper bound of the timestamp - range. If omitted, defaults to "infinity" (no upper bound). - """ - - def __init__(self, start=None, end=None): - self.start = start - self.end = end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start == self.start and - other.end == self.end) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`TimestampRange` to a protobuf. - - :rtype: :class:`.data_pb2.TimestampRange` - :returns: The converted current object. - """ - timestamp_range_kwargs = {} - if self.start is not None: - timestamp_range_kwargs['start_timestamp_micros'] = ( - _timestamp_to_microseconds(self.start)) - if self.end is not None: - timestamp_range_kwargs['end_timestamp_micros'] = ( - _timestamp_to_microseconds(self.end)) - return data_pb2.TimestampRange(**timestamp_range_kwargs) - - -class ColumnRange(object): - """A range of columns to restrict to in a row filter. - - Both the start and end column can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type column_family_id: str - :param column_family_id: The column family that contains the columns. Must - be of the form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type start_column: bytes - :param start_column: The start of the range of columns. If no value is - used, it is interpreted as the empty string - (inclusive) by the backend. - - :type end_column: bytes - :param end_column: The end of the range of columns. If no value is used, it - is interpreted as the infinite string (exclusive) by the - backend. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start column should be - included in the range (or excluded). - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end column should be - included in the range (or excluded). - """ - - def __init__(self, column_family_id, start_column=None, end_column=None, - inclusive_start=True, inclusive_end=True): - self.column_family_id = column_family_id - self.start_column = start_column - self.end_column = end_column - self.inclusive_start = inclusive_start - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.column_family_id == self.column_family_id and - other.start_column == self.start_column and - other.end_column == self.end_column and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`ColumnRange` to a protobuf. - - :rtype: :class:`.data_pb2.ColumnRange` - :returns: The converted current object. - """ - column_range_kwargs = {'family_name': self.column_family_id} - if self.start_column is not None: - if self.inclusive_start: - key = 'start_qualifier_inclusive' - else: - key = 'start_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.start_column) - if self.end_column is not None: - if self.inclusive_end: - key = 'end_qualifier_inclusive' - else: - key = 'end_qualifier_exclusive' - column_range_kwargs[key] = _to_bytes(self.end_column) - return data_pb2.ColumnRange(**column_range_kwargs) - - -class CellValueRange(object): - """A range of values to restrict to in a row filter. - - Will only match cells that have values in this range. - - Both the start and end value can be included or excluded in the range. - By default, we include them both, but this can be changed with optional - flags. - - :type start_value: bytes - :param start_value: The start of the range of values. If no value is - used, it is interpreted as the empty string - (inclusive) by the backend. - - :type end_value: bytes - :param end_value: The end of the range of values. If no value is used, it - is interpreted as the infinite string (exclusive) by the - backend. - - :type inclusive_start: bool - :param inclusive_start: Boolean indicating if the start value should be - included in the range (or excluded). - - :type inclusive_end: bool - :param inclusive_end: Boolean indicating if the end value should be - included in the range (or excluded). - """ - - def __init__(self, start_value=None, end_value=None, - inclusive_start=True, inclusive_end=True): - self.start_value = start_value - self.end_value = end_value - self.inclusive_start = inclusive_start - self.inclusive_end = inclusive_end - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.start_value == self.start_value and - other.end_value == self.end_value and - other.inclusive_start == self.inclusive_start and - other.inclusive_end == self.inclusive_end) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`CellValueRange` to a protobuf. - - :rtype: :class:`.data_pb2.ValueRange` - :returns: The converted current object. - """ - value_range_kwargs = {} - if self.start_value is not None: - if self.inclusive_start: - key = 'start_value_inclusive' - else: - key = 'start_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.start_value) - if self.end_value is not None: - if self.inclusive_end: - key = 'end_value_inclusive' - else: - key = 'end_value_exclusive' - value_range_kwargs[key] = _to_bytes(self.end_value) - return data_pb2.ValueRange(**value_range_kwargs) - - -class RowFilterChain(object): - """Chain of row filters. - - Sends rows through several filters in sequence. The filters are "chained" - together to process a row. After the first filter is applied, the second - is applied to the filtered output and so on for subsequent filters. - - :type filters: list - :param filters: List of :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` and/or - :class:`ConditionalRowFilter` - """ - - def __init__(self, filters=None): - self.filters = filters - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.filters == self.filters - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`RowFilterChain` to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - chain = data_pb2.RowFilter.Chain( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(chain=chain) - - -class RowFilterUnion(object): - """Union of row filters. - - Sends rows through several filters simultaneously, then - merges / interleaves all the filtered results together. - - If multiple cells are produced with the same column and timestamp, - they will all appear in the output row in an unspecified mutual order. - - :type filters: list - :param filters: List of :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` and/or - :class:`ConditionalRowFilter` - """ - - def __init__(self, filters=None): - self.filters = filters - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other.filters == self.filters - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`RowFilterUnion` to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - interleave = data_pb2.RowFilter.Interleave( - filters=[row_filter.to_pb() for row_filter in self.filters]) - return data_pb2.RowFilter(interleave=interleave) - - -class ConditionalRowFilter(object): - """Conditional filter - - Executes one of two filters based on another filter. If the ``base_filter`` - returns any cells in the row, then ``true_filter`` is executed. If not, - then ``false_filter`` is executed. - - .. note:: - - The ``base_filter`` does not execute atomically with the true and false - filters, which may lead to inconsistent or unexpected results. - - Additionally, executing a :class:`ConditionalRowFilter` has poor - performance on the server, especially when ``false_filter`` is set. - - :type base_filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` - :param base_filter: The filter to condition on before executing the - true/false filters. - - :type true_filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or :class:`ConditionalRowFilter` - :param true_filter: (Optional) The filter to execute if there are any cells - matching ``base_filter``. If not provided, no results - will be returned in the true case. - - :type false_filter: :class:`RowFilter`, :class:`RowFilterChain`, - :class:`RowFilterUnion` or - :class:`ConditionalRowFilter` - :param false_filter: (Optional) The filter to execute if there are no cells - matching ``base_filter``. If not provided, no results - will be returned in the false case. - """ - - def __init__(self, base_filter, true_filter=None, false_filter=None): - self.base_filter = base_filter - self.true_filter = true_filter - self.false_filter = false_filter - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.base_filter == self.base_filter and - other.true_filter == self.true_filter and - other.false_filter == self.false_filter) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_pb(self): - """Converts the :class:`ConditionalRowFilter` to a protobuf. - - :rtype: :class:`.data_pb2.RowFilter` - :returns: The converted current object. - """ - condition_kwargs = {'predicate_filter': self.base_filter.to_pb()} - if self.true_filter is not None: - condition_kwargs['true_filter'] = self.true_filter.to_pb() - if self.false_filter is not None: - condition_kwargs['false_filter'] = self.false_filter.to_pb() - condition = data_pb2.RowFilter.Condition(**condition_kwargs) - return data_pb2.RowFilter(condition=condition) - - -def _parse_rmw_row_response(row_response): - """Parses the response to a ``ReadModifyWriteRow`` request. - - :type row_response: :class:`.data_pb2.Row` - :param row_response: The response row (with only modified cells) from a - ``ReadModifyWriteRow`` request. - - :rtype: dict - :returns: The new contents of all modified cells. Returned as a - dictionary of column families, each of which holds a - dictionary of columns. Each column contains a list of cells - modified. Each cell is represented with a two-tuple with the - value (in bytes) and the timestamp for the cell. For example: - - .. code:: python - - { - u'col-fam-id': { - b'col-name1': [ - (b'cell-val', datetime.datetime(...)), - (b'cell-val-newer', datetime.datetime(...)), - ], - b'col-name2': [ - (b'altcol-cell-val', datetime.datetime(...)), - ], - }, - u'col-fam-id2': { - b'col-name3-but-other-fam': [ - (b'foo', datetime.datetime(...)), - ], - }, - } - """ - result = {} - for column_family in row_response.families: - column_family_id, curr_family = _parse_family_pb(column_family) - result[column_family_id] = curr_family - return result diff --git a/gcloud_bigtable/row_data.py b/gcloud_bigtable/row_data.py deleted file mode 100644 index 3f9a80f..0000000 --- a/gcloud_bigtable/row_data.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Container for Google Cloud Bigtable Cells and Streaming Row Contents.""" - - -import copy -import six - -from gcloud_bigtable._helpers import _microseconds_to_timestamp -from gcloud_bigtable._helpers import _to_bytes - - -class Cell(object): - """Representation of a Google Cloud Bigtable Cell. - - :type value: bytes - :param value: The value stored in the cell. - - :type timestamp: :class:`datetime.datetime` - :param timestamp: The timestamp when the cell was stored. - - :type labels: list - :param labels: (Optional) List of strings. Labels applied to the cell. - """ - - def __init__(self, value, timestamp, labels=()): - self.value = value - self.timestamp = timestamp - self.labels = list(labels) - - @classmethod - def from_pb(cls, cell_pb): - """Create a new cell from a Cell protobuf. - - :type cell_pb: :class:`._generated.bigtable_data_pb2.Cell` - :param cell_pb: The protobuf to convert. - - :rtype: :class:`Cell` - :returns: The cell corresponding to the protobuf. - """ - timestamp = _microseconds_to_timestamp(cell_pb.timestamp_micros) - if cell_pb.labels: - return cls(cell_pb.value, timestamp, labels=cell_pb.labels) - else: - return cls(cell_pb.value, timestamp) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.value == self.value and - other.timestamp == self.timestamp and - other.labels == self.labels) - - def __ne__(self, other): - return not self.__eq__(other) - - -class PartialRowData(object): - """Representation of partial row in a Google Cloud Bigtable Table. - - These are expected to be updated directly from a - :class:`._generated.bigtable_service_messages_pb2.ReadRowsResponse` - - :type row_key: bytes - :param row_key: The key for the row holding the (partial) data. - """ - - def __init__(self, row_key): - self._row_key = row_key - self._cells = {} - self._committed = False - self._chunks_encountered = False - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other._row_key == self._row_key and - other._committed == self._committed and - other._chunks_encountered == self._chunks_encountered and - other._cells == self._cells) - - def __ne__(self, other): - return not self.__eq__(other) - - def to_dict(self): - """Convert the cells to a dictionary. - - This is intended to be used with HappyBase, so the column family and - column qualiers are combined (with ``:``). - - :rtype: dict - :returns: Dictionary containing all the data in the cells of this row. - """ - result = {} - for column_family_id, columns in six.iteritems(self._cells): - for column_qual, cells in six.iteritems(columns): - key = (_to_bytes(column_family_id) + b':' + - _to_bytes(column_qual)) - result[key] = cells - return result - - @property - def cells(self): - """Property returning all the cells accumulated on this partial row. - - :rtype: dict - :returns: Dictionary of the :class:`Cell` objects accumulated. This - dictionary has two-levels of keys (first for column families - and second for column names/qualifiers within a family). For - a given column, a list of :class:`Cell` objects is stored. - """ - return copy.deepcopy(self._cells) - - @property - def row_key(self): - """Getter for the current (partial) row's key. - - :rtype: bytes - :returns: The current (partial) row's key. - """ - return self._row_key - - @property - def committed(self): - """Getter for the committed status of the (partial) row. - - :rtype: bool - :returns: The committed status of the (partial) row. - """ - return self._committed - - def clear(self): - """Clears all cells that have been added.""" - self._committed = False - self._chunks_encountered = False - self._cells.clear() - - def _handle_commit_row(self, chunk, index, last_chunk_index): - """Handles a ``commit_row`` chunk. - - :type chunk: ``ReadRowsResponse.Chunk`` - :param chunk: The chunk being handled. - - :type index: int - :param index: The current index of the chunk. - - :type last_chunk_index: int - :param last_chunk_index: The index of the last chunk. - - :raises: :class:`ValueError ` if the value of - ``commit_row`` is :data:`False` or if the chunk passed is not - the last chunk in a response. - """ - # NOTE: We assume the caller has checked that the ``ONEOF`` property - # for ``chunk`` is ``commit_row``. - if not chunk.commit_row: - raise ValueError('Received commit_row that was False.') - - if index != last_chunk_index: - raise ValueError('Commit row chunk was not the last chunk') - else: - self._committed = True - - def _handle_reset_row(self, chunk): - """Handles a ``reset_row`` chunk. - - :type chunk: ``ReadRowsResponse.Chunk`` - :param chunk: The chunk being handled. - - :raises: :class:`ValueError ` if the value of - ``reset_row`` is :data:`False` - """ - # NOTE: We assume the caller has checked that the ``ONEOF`` property - # for ``chunk`` is ``reset_row``. - if not chunk.reset_row: - raise ValueError('Received reset_row that was False.') - - self.clear() - - def _handle_row_contents(self, chunk): - """Handles a ``row_contents`` chunk. - - :type chunk: ``ReadRowsResponse.Chunk`` - :param chunk: The chunk being handled. - """ - # NOTE: We assume the caller has checked that the ``ONEOF`` property - # for ``chunk`` is ``row_contents``. - - # chunk.row_contents is ._generated.bigtable_data_pb2.Family - column_family_id = chunk.row_contents.name - column_family_dict = self._cells.setdefault(column_family_id, {}) - for column in chunk.row_contents.columns: - cells = [Cell.from_pb(cell) for cell in column.cells] - - column_name = column.qualifier - column_cells = column_family_dict.setdefault(column_name, []) - column_cells.extend(cells) - - def update_from_read_rows(self, read_rows_response_pb): - """Updates the current row from a ``ReadRows`` response. - - :type read_rows_response_pb: - :class:`._generated.bigtable_service_messages_pb2.ReadRowsResponse` - :param read_rows_response_pb: A response streamed back as part of a - ``ReadRows`` request. - - :raises: :class:`ValueError ` if the current - partial row has already been committed, if the row key on the - response doesn't match the current one or if there is a chunk - encountered with an unexpected ``ONEOF`` protobuf property. - """ - if self._committed: - raise ValueError('The row has been committed') - - if read_rows_response_pb.row_key != self.row_key: - raise ValueError('Response row key (%r) does not match current ' - 'one (%r).' % (read_rows_response_pb.row_key, - self.row_key)) - - last_chunk_index = len(read_rows_response_pb.chunks) - 1 - for index, chunk in enumerate(read_rows_response_pb.chunks): - chunk_property = chunk.WhichOneof('chunk') - if chunk_property == 'row_contents': - self._handle_row_contents(chunk) - elif chunk_property == 'reset_row': - self._handle_reset_row(chunk) - elif chunk_property == 'commit_row': - self._handle_commit_row(chunk, index, last_chunk_index) - else: - # NOTE: This includes chunk_property == None since we always - # want a value to be set - raise ValueError('Unexpected chunk property: %s' % ( - chunk_property,)) - - self._chunks_encountered = True - - -class PartialRowsData(object): - """Convenience wrapper for consuming a ``ReadRows`` streaming response. - - :type response_iterator: - :class:`grpc.framework.alpha._reexport._CancellableIterator` - :param response_iterator: A streaming iterator returned from a - ``ReadRows`` request. - """ - - def __init__(self, response_iterator): - # We expect an iterator of `data_messages_pb2.ReadRowsResponse` - self._response_iterator = response_iterator - self._rows = {} - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return other._response_iterator == self._response_iterator - - def __ne__(self, other): - return not self.__eq__(other) - - @property - def rows(self): - """Property returning all rows accumulated from the stream. - - :rtype: dict - :returns: Dictionary of :class:`PartialRowData`. - """ - # NOTE: To avoid duplicating large objects, this is just the - # mutable private data. - return self._rows - - def cancel(self): - """Cancels the iterator, closing the stream.""" - self._response_iterator.cancel() - - def consume_next(self): - """Consumes the next ``ReadRowsResponse`` from the stream. - - Parses the response and stores it as a :class:`PartialRowData` - in a dictionary owned by this object. - - :raises: :class:`StopIteration ` if the - response iterator has no more responses to stream. - """ - read_rows_response = self._response_iterator.next() - row_key = read_rows_response.row_key - partial_row = self._rows.get(row_key) - if partial_row is None: - partial_row = self._rows[row_key] = PartialRowData(row_key) - # NOTE: This is not atomic in the case of failures. - partial_row.update_from_read_rows(read_rows_response) - - def consume_all(self, max_loops=None): - """Consume the streamed responses until there are no more. - - This simply calls :meth:`consume_next` until there are no - more to consume. - - :type max_loops: int - :param max_loops: (Optional) Maximum number of times to try to consume - an additional ``ReadRowsResponse``. You can use this - to avoid long wait times. - """ - curr_loop = 0 - if max_loops is None: - max_loops = float('inf') - while curr_loop < max_loops: - curr_loop += 1 - try: - self.consume_next() - except StopIteration: - break diff --git a/gcloud_bigtable/table.py b/gcloud_bigtable/table.py deleted file mode 100644 index b5fbec1..0000000 --- a/gcloud_bigtable/table.py +++ /dev/null @@ -1,497 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""User friendly container for Google Cloud Bigtable Table.""" - - -from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 -from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as data_messages_pb2) -from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) -from gcloud_bigtable._helpers import _to_bytes -from gcloud_bigtable.column_family import ColumnFamily -from gcloud_bigtable.column_family import _gc_rule_from_pb -from gcloud_bigtable.row import Row -from gcloud_bigtable.row_data import PartialRowData -from gcloud_bigtable.row_data import PartialRowsData - - -class Table(object): - """Representation of a Google Cloud Bigtable Table. - - .. note:: - - We don't define any properties on a table other than the name. As - the proto says, in a request: - - The ``name`` field of the Table and all of its ColumnFamilies must - be left blank, and will be populated in the response. - - This leaves only the ``current_operation`` and ``granularity`` - fields. The ``current_operation`` is only used for responses while - ``granularity`` is an enum with only one value. - - We can use a :class:`Table` to: - - * :meth:`create` the table - * :meth:`rename` the table - * :meth:`delete` the table - * :meth:`list_column_families` in the table - - :type table_id: str - :param table_id: The ID of the table. - - :type cluster: :class:`.cluster.Cluster` - :param cluster: The cluster that owns the table. - """ - - def __init__(self, table_id, cluster): - self.table_id = table_id - self._cluster = cluster - - @property - def cluster(self): - """Getter for table's cluster. - - :rtype: :class:`.cluster.Cluster` - :returns: The cluster stored on the table. - """ - return self._cluster - - @property - def client(self): - """Getter for table's client. - - :rtype: :class:`.client.Client` - :returns: The client that owns this table. - """ - return self.cluster.client - - @property - def timeout_seconds(self): - """Getter for table's default timeout seconds. - - :rtype: int - :returns: The timeout seconds default stored on the table's client. - """ - return self._cluster.timeout_seconds - - @property - def name(self): - """Table name used in requests. - - .. note:: - - This property will not change if ``table_id`` does not, but the - return value is not cached. - - The table name is of the form - - ``"projects/../zones/../clusters/../tables/{table_id}"`` - - :rtype: str - :returns: The table name. - """ - return self.cluster.name + '/tables/' + self.table_id - - def column_family(self, column_family_id, gc_rule=None): - """Factory to create a column family associated with this table. - - :type column_family_id: str - :param column_family_id: The ID of the column family. Must be of the - form ``[_a-zA-Z0-9][-_.a-zA-Z0-9]*``. - - :type gc_rule: :class:`.column_family.GarbageCollectionRule`, - :class:`.column_family.GarbageCollectionRuleUnion` or - :class:`.column_family.GarbageCollectionRuleIntersection` - :param gc_rule: (Optional) The garbage collection settings for this - column family. - - :rtype: :class:`.column_family.ColumnFamily` - :returns: A column family owned by this table. - """ - return ColumnFamily(column_family_id, self, gc_rule=gc_rule) - - def row(self, row_key, filter_=None): - """Factory to create a row associated with this table. - - :type row_key: bytes - :param row_key: The key for the row being created. - - :type filter_: :class:`.RowFilter`, - :class:`.RowFilterChain`, - :class:`.RowFilterUnion`, or - :class:`.ConditionalRowFilter` - :param filter_: (Optional) Filter to be used for conditional mutations. - See :class:`.Row` for more details. - - :rtype: :class:`.Row` - :returns: A row owned by this table. - """ - return Row(row_key, self, filter_=filter_) - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - return (other.table_id == self.table_id and - other.cluster == self.cluster) - - def __ne__(self, other): - return not self.__eq__(other) - - def create(self, initial_split_keys=None, timeout_seconds=None): - """Creates this table. - - .. note:: - - Though a :class:`._generated.bigtable_table_data_pb2.Table` is also - allowed (as the ``table`` property) in a create table request, we - do not support it in this method. As mentioned in the - :class:`Table` docstring, the name is the only useful property in - the table proto. - - .. note:: - - A create request returns a - :class:`._generated.bigtable_table_data_pb2.Table` but we don't use - this response. The proto definition allows for the inclusion of a - ``current_operation`` in the response, but in example usage so far, - it seems the Bigtable API does not return any operation. - - :type initial_split_keys: list - :param initial_split_keys: (Optional) List of row keys that will be - used to initially split the table into - several tablets (Tablets are similar to - HBase regions). Given two split keys, - ``"s1"`` and ``"s2"``, three tablets will be - created, spanning the key ranges: - ``[, s1)``, ``[s1, s2)``, ``[s2, )``. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - """ - request_pb = messages_pb2.CreateTableRequest( - initial_split_keys=initial_split_keys or [], - name=self.cluster.name, - table_id=self.table_id, - ) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.CreateTable.async(request_pb, - timeout_seconds) - # We expect a `._generated.bigtable_table_data_pb2.Table` - response.result() - - def rename(self, new_table_id, timeout_seconds=None): - """Rename this table. - - .. note:: - - This cannot be used to move tables between clusters, - zones, or projects. - - .. note:: - - The Bigtable Table Admin API currently returns - - ``BigtableTableService.RenameTable is not yet implemented`` - - when this method is used. It's unclear when this method will - actually be supported by the API. - - :type new_table_id: str - :param new_table_id: The new name table ID. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - """ - request_pb = messages_pb2.RenameTableRequest( - name=self.name, - new_id=new_table_id, - ) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.RenameTable.async(request_pb, - timeout_seconds) - # We expect a `._generated.empty_pb2.Empty` - response.result() - - self.table_id = new_table_id - - def delete(self, timeout_seconds=None): - """Delete this table. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - """ - request_pb = messages_pb2.DeleteTableRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.DeleteTable.async(request_pb, - timeout_seconds) - # We expect a `._generated.empty_pb2.Empty` - response.result() - - def list_column_families(self, timeout_seconds=None): - """Check if this table exists. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - - :rtype: dictionary with string as keys and - :class:`.column_family.ColumnFamily` as values - :returns: List of column families attached to this table. - :raises: :class:`ValueError ` if the column - family name from the response does not agree with the computed - name from the column family ID. - """ - request_pb = messages_pb2.GetTableRequest(name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response = self.client._table_stub.GetTable.async(request_pb, - timeout_seconds) - # We expect a `._generated.bigtable_table_data_pb2.Table` - table_pb = response.result() - - result = {} - for column_family_id, value_pb in table_pb.column_families.items(): - gc_rule = _gc_rule_from_pb(value_pb.gc_rule) - column_family = self.column_family(column_family_id, - gc_rule=gc_rule) - if column_family.name != value_pb.name: - raise ValueError('Column family name %s does not agree with ' - 'name from request: %s.' % ( - column_family.name, value_pb.name)) - result[column_family_id] = column_family - return result - - def read_row(self, row_key, filter_=None, timeout_seconds=None): - """Read a single row from this table. - - :type row_key: bytes - :param row_key: The key of the row to read from. - - :type filter_: :class:`.row.RowFilter`, :class:`.row.RowFilterChain`, - :class:`.row.RowFilterUnion` or - :class:`.row.ConditionalRowFilter` - :param filter_: (Optional) The filter to apply to the contents of the - row. If unset, returns the entire row. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - - :rtype: :class:`.PartialRowData`, :data:`NoneType ` - :returns: The contents of the row if any chunks were returned in - the response, otherwise :data:`None`. - :raises: :class:`ValueError ` if a commit row - chunk is never encountered. - """ - request_pb = _create_row_request(self.name, row_key=row_key, - filter_=filter_) - timeout_seconds = timeout_seconds or self.timeout_seconds - response_iterator = self.client._data_stub.ReadRows(request_pb, - timeout_seconds) - # We expect an iterator of `data_messages_pb2.ReadRowsResponse` - result = PartialRowData(row_key) - for read_rows_response in response_iterator: - result.update_from_read_rows(read_rows_response) - - # Make sure the result actually contains data. - if not result._chunks_encountered: - return None - # Make sure the result was committed by the back-end. - if not result.committed: - raise ValueError('The row remains partial / is not committed.') - return result - - def read_rows(self, start_key=None, end_key=None, - allow_row_interleaving=None, limit=None, filter_=None, - timeout_seconds=None): - """Read rows from this table. - - :type start_key: bytes - :param start_key: (Optional) The beginning of a range of row keys to - read from. The range will include ``start_key``. If - left empty, will be interpreted as the empty string. - - :type end_key: bytes - :param end_key: (Optional) The end of a range of row keys to read from. - The range will not include ``end_key``. If left empty, - will be interpreted as an infinite string. - - :type filter_: :class:`.row.RowFilter`, :class:`.row.RowFilterChain`, - :class:`.row.RowFilterUnion` or - :class:`.row.ConditionalRowFilter` - :param filter_: (Optional) The filter to apply to the contents of the - specified row(s). If unset, reads every column in - each row. - - :type allow_row_interleaving: bool - :param allow_row_interleaving: (Optional) By default, rows are read - sequentially, producing results which - are guaranteed to arrive in increasing - row order. Setting - ``allow_row_interleaving`` to - :data:`True` allows multiple rows to be - interleaved in the response stream, - which increases throughput but breaks - this guarantee, and may force the - client to use more memory to buffer - partially-received rows. - - :type limit: int - :param limit: (Optional) The read will terminate after committing to N - rows' worth of results. The default (zero) is to return - all results. Note that if ``allow_row_interleaving`` is - set to :data:`True`, partial results may be returned for - more than N rows. However, only N ``commit_row`` chunks - will be sent. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - - :rtype: :class:`.PartialRowsData` - :returns: A :class:`.PartialRowsData` convenience wrapper for consuming - the streamed results. - """ - request_pb = _create_row_request( - self.name, start_key=start_key, end_key=end_key, filter_=filter_, - allow_row_interleaving=allow_row_interleaving, limit=limit) - timeout_seconds = timeout_seconds or self.timeout_seconds - response_iterator = self.client._data_stub.ReadRows(request_pb, - timeout_seconds) - # We expect an iterator of `data_messages_pb2.ReadRowsResponse` - return PartialRowsData(response_iterator) - - def sample_row_keys(self, timeout_seconds=None): - """Read a sample of row keys in the table. - - The returned row keys will delimit contiguous sections of the table of - approximately equal size, which can be used to break up the data for - distributed tasks like mapreduces. - - The elements in the iterator are a SampleRowKeys response and they have - the properties ``offset_bytes`` and ``row_key``. They occur in sorted - order. The table might have contents before the first row key in the - list and after the last one, but a key containing the empty string - indicates "end of table" and will be the last response given, if - present. - - .. note:: - - Row keys in this list may not have ever been written to or read - from, and users should therefore not make any assumptions about the - row key structure that are specific to their use case. - - The ``offset_bytes`` field on a response indicates the approximate - total storage space used by all rows in the table which precede - ``row_key``. Buffering the contents of all rows between two subsequent - samples would require space roughly equal to the difference in their - ``offset_bytes`` fields. - - :type timeout_seconds: int - :param timeout_seconds: Number of seconds for request time-out. - If not passed, defaults to value set on table. - - :rtype: :class:`grpc.framework.alpha._reexport._CancellableIterator` - :returns: A cancel-able iterator. Can be consumed by calling ``next()`` - or by casting to a :class:`list` and can be cancelled by - calling ``cancel()``. - """ - request_pb = data_messages_pb2.SampleRowKeysRequest( - table_name=self.name) - timeout_seconds = timeout_seconds or self.timeout_seconds - response_iterator = self.client._data_stub.SampleRowKeys( - request_pb, timeout_seconds) - return response_iterator - - -def _create_row_request(table_name, row_key=None, start_key=None, end_key=None, - filter_=None, allow_row_interleaving=None, limit=None): - """Creates a request to read rows in a table. - - :type table_name: str - :param table_name: The name of the table to read from. - - :type row_key: bytes - :param row_key: (Optional) The key of a specific row to read from. - - :type start_key: bytes - :param start_key: (Optional) The beginning of a range of row keys to - read from. The range will include ``start_key``. If - left empty, will be interpreted as the empty string. - - :type end_key: bytes - :param end_key: (Optional) The end of a range of row keys to read from. - The range will not include ``end_key``. If left empty, - will be interpreted as an infinite string. - - :type filter_: :class:`.row.RowFilter`, :class:`.row.RowFilterChain`, - :class:`.row.RowFilterUnion` or - :class:`.row.ConditionalRowFilter` - :param filter_: (Optional) The filter to apply to the contents of the - specified row(s). If unset, reads the entire table. - - :type allow_row_interleaving: bool - :param allow_row_interleaving: (Optional) By default, rows are read - sequentially, producing results which are - guaranteed to arrive in increasing row - order. Setting - ``allow_row_interleaving`` to - :data:`True` allows multiple rows to be - interleaved in the response stream, - which increases throughput but breaks - this guarantee, and may force the - client to use more memory to buffer - partially-received rows. - - :type limit: int - :param limit: (Optional) The read will terminate after committing to N - rows' worth of results. The default (zero) is to return - all results. Note that if ``allow_row_interleaving`` is - set to :data:`True`, partial results may be returned for - more than N rows. However, only N ``commit_row`` chunks - will be sent. - - :rtype: :class:`data_messages_pb2.ReadRowsRequest` - :returns: The ``ReadRowsRequest`` protobuf corresponding to the inputs. - :raises: :class:`ValueError ` if both - ``row_key`` and one of ``start_key`` and ``end_key`` are set - """ - request_kwargs = {'table_name': table_name} - if (row_key is not None and - (start_key is not None or end_key is not None)): - raise ValueError('Row key and row range cannot be ' - 'set simultaneously') - if row_key is not None: - request_kwargs['row_key'] = _to_bytes(row_key) - if start_key is not None or end_key is not None: - range_kwargs = {} - if start_key is not None: - range_kwargs['start_key'] = _to_bytes(start_key) - if end_key is not None: - range_kwargs['end_key'] = _to_bytes(end_key) - row_range = data_pb2.RowRange(**range_kwargs) - request_kwargs['row_range'] = row_range - if filter_ is not None: - request_kwargs['filter'] = filter_.to_pb() - if allow_row_interleaving is not None: - request_kwargs['allow_row_interleaving'] = allow_row_interleaving - if limit is not None: - request_kwargs['num_rows_limit'] = limit - - return data_messages_pb2.ReadRowsRequest(**request_kwargs) diff --git a/gcloud_bigtable/test___init__.py b/gcloud_bigtable/test___init__.py deleted file mode 100644 index e9d9afc..0000000 --- a/gcloud_bigtable/test___init__.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestGRPCImportFailure(unittest2.TestCase): - - _MODULES_TO_EDIT = ('gcloud_bigtable', 'grpc', - 'grpc._adapter', 'grpc._adapter._c') - - @classmethod - def setUpClass(cls): - import imp - mod_name = 'gcloud_bigtable' - # H/T: http://pymotw.com/2/imp/ - cls._load_module_args = (mod_name,) + imp.find_module(mod_name) - - @classmethod - def tearDownClass(cls): - del cls._load_module_args - - def _module_patch_helper(self, function_to_test): - import sys - - removed_mods = {} - for mod_name in self._MODULES_TO_EDIT: - removed_mods[mod_name] = sys.modules.get(mod_name) - - try: - sys.modules.pop('gcloud_bigtable', None) - # Test method should re-import gcloud_bigtable - function_to_test() - finally: - for mod_name, value in removed_mods.items(): - sys.modules[mod_name] = value - - def test_success(self): - import imp - import sys - import types - - TEST_CASE = self - - def function_to_test(): - c_mod = types.ModuleType('grpc._adapter._c') - sys.modules['grpc._adapter._c'] = c_mod - - adapter_mod = types.ModuleType('grpc._adapter') - adapter_mod._c = c_mod - sys.modules['grpc._adapter'] = adapter_mod - - grpc_mod = types.ModuleType('grpc') - grpc_mod._adapter = adapter_mod - sys.modules['grpc'] = grpc_mod - - # Re-import gcloud_bigtable and check our custom _c module. - gcloud_bigtable = imp.load_module(*TEST_CASE._load_module_args) - TEST_CASE.assertTrue(gcloud_bigtable._c is c_mod) - - self._module_patch_helper(function_to_test) - - @staticmethod - def _create_fake_grpc_adapter_package(contents): - import os - import tempfile - - temp_dir = tempfile.mkdtemp() - - curr_dir = os.path.join(temp_dir, 'grpc') - os.mkdir(curr_dir) - with open(os.path.join(curr_dir, '__init__.py'), 'wb') as file_obj: - file_obj.write(b'') - - curr_dir = os.path.join(curr_dir, '_adapter') - os.mkdir(curr_dir) - with open(os.path.join(curr_dir, '__init__.py'), 'wb') as file_obj: - file_obj.write(b'') - - filename = os.path.join(curr_dir, '_c.py') - with open(filename, 'wb') as file_obj: - file_obj.write(contents) - - return temp_dir - - def _import_fail_helper(self, orig_exc_message): - import imp - import sys - - # Be sure the message contains libgrpc.so - c_module_contents = 'raise ImportError(%r)\n' % (orig_exc_message,) - c_module_contents = c_module_contents.encode('ascii') - temp_dir = self._create_fake_grpc_adapter_package(c_module_contents) - - TEST_CASE = self - - def function_to_test(): - sys.path.insert(0, temp_dir) - try: - sys.modules.pop('grpc') - sys.modules.pop('grpc._adapter') - sys.modules.pop('grpc._adapter._c') - - try: - imp.load_module(*TEST_CASE._load_module_args) - except ImportError as exc: - if 'libgrpc.so' in orig_exc_message: - TEST_CASE.assertNotEqual(str(exc), orig_exc_message) - else: - TEST_CASE.assertEqual(str(exc), orig_exc_message) - finally: - sys.path.remove(temp_dir) - - self._module_patch_helper(function_to_test) - - def test_system_library_cause(self): - # Be sure the message contains libgrpc.so - orig_exc_message = 'bad libgrpc.so' - self._import_fail_helper(orig_exc_message) - - def test_non_system_library_cause(self): - # Be sure the message does not contain libgrpc.so - orig_exc_message = 'Other cause of error' - self._import_fail_helper(orig_exc_message) diff --git a/gcloud_bigtable/test__helpers.py b/gcloud_bigtable/test__helpers.py deleted file mode 100644 index ef98615..0000000 --- a/gcloud_bigtable/test__helpers.py +++ /dev/null @@ -1,530 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestMetadataTransformer(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable._helpers import MetadataTransformer - return MetadataTransformer - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - from gcloud_bigtable.client import DATA_SCOPE - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - project = 'PROJECT' - user_agent = 'USER_AGENT' - client = Client(project=project, credentials=credentials, - user_agent=user_agent) - transformer = self._makeOne(client) - self.assertTrue(transformer._credentials is scoped_creds) - self.assertEqual(transformer._user_agent, user_agent) - self.assertEqual(credentials._called, [ - ('create_scoped', ([DATA_SCOPE],), {}), - ]) - - def test___call__(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - from gcloud_bigtable.client import DATA_SCOPE - from gcloud_bigtable.client import DEFAULT_USER_AGENT - - access_token_expected = 'FOOBARBAZ' - - class _ReturnVal(object): - access_token = access_token_expected - - scoped_creds = _MockWithAttachedMethods(_ReturnVal) - credentials = _MockWithAttachedMethods(scoped_creds) - project = 'PROJECT' - client = Client(project=project, credentials=credentials) - - transformer = self._makeOne(client) - result = transformer(None) - self.assertEqual( - result, - [ - ('Authorization', 'Bearer ' + access_token_expected), - ('User-agent', DEFAULT_USER_AGENT), - ]) - self.assertEqual(credentials._called, [ - ('create_scoped', ([DATA_SCOPE],), {}), - ]) - self.assertEqual(scoped_creds._called, [('get_access_token', (), {})]) - - -class Test__pb_timestamp_to_datetime(unittest2.TestCase): - - def _callFUT(self, timestamp): - from gcloud_bigtable._helpers import _pb_timestamp_to_datetime - return _pb_timestamp_to_datetime(timestamp) - - def test_it(self): - import datetime - import pytz - from gcloud_bigtable._generated.timestamp_pb2 import Timestamp - - # Epoch is midnight on January 1, 1970 ... - dt_stamp = datetime.datetime(1970, month=1, day=1, hour=0, - minute=1, second=1, microsecond=1234, - tzinfo=pytz.utc) - # ... so 1 minute and 1 second after is 61 seconds and 1234 - # microseconds is 1234000 nanoseconds. - timestamp = Timestamp(seconds=61, nanos=1234000) - self.assertEqual(self._callFUT(timestamp), dt_stamp) - - -class Test__require_pb_property(unittest2.TestCase): - - def _callFUT(self, message_pb, property_name, value): - from gcloud_bigtable._helpers import _require_pb_property - return _require_pb_property(message_pb, property_name, value) - - def test_it(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = 119 - cluster_pb = data_pb2.Cluster(serve_nodes=serve_nodes) - result = self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - self.assertEqual(result, serve_nodes) - - def test_with_null_value_passed_in(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = None - actual_serve_nodes = 119 - cluster_pb = data_pb2.Cluster(serve_nodes=actual_serve_nodes) - result = self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - self.assertEqual(result, actual_serve_nodes) - - def test_with_value_unset_on_pb(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = 119 - cluster_pb = data_pb2.Cluster() - with self.assertRaises(ValueError): - self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - - def test_with_values_disagreeing(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - serve_nodes = 119 - other_serve_nodes = 1000 - self.assertNotEqual(serve_nodes, other_serve_nodes) - cluster_pb = data_pb2.Cluster(serve_nodes=other_serve_nodes) - with self.assertRaises(ValueError): - self._callFUT(cluster_pb, 'serve_nodes', serve_nodes) - - -class Test__parse_pb_any_to_native(unittest2.TestCase): - - def _callFUT(self, any_val, expected_type=None): - from gcloud_bigtable._helpers import _parse_pb_any_to_native - return _parse_pb_any_to_native(any_val, expected_type=expected_type) - - def test_it(self): - from gcloud_bigtable._generated import any_pb2 - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - type_url = 'type.googleapis.com/' + data_pb2._CELL.full_name - fake_type_url_map = {type_url: data_pb2.Cell} - - cell = data_pb2.Cell( - timestamp_micros=0, - value=b'foobar', - ) - any_val = any_pb2.Any( - type_url=type_url, - value=cell.SerializeToString(), - ) - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - result = self._callFUT(any_val) - - self.assertEqual(result, cell) - - def test_unknown_type_url(self): - from gcloud_bigtable._generated import any_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - fake_type_url_map = {} - any_val = any_pb2.Any() - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - with self.assertRaises(KeyError): - self._callFUT(any_val) - - def test_disagreeing_type_url(self): - from gcloud_bigtable._generated import any_pb2 - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - type_url1 = 'foo' - type_url2 = 'bar' - fake_type_url_map = {type_url1: None} - any_val = any_pb2.Any(type_url=type_url2) - with _Monkey(MUT, _TYPE_URL_MAP=fake_type_url_map): - with self.assertRaises(ValueError): - self._callFUT(any_val, expected_type=type_url1) - - -class Test__timedelta_to_duration_pb(unittest2.TestCase): - - def _callFUT(self, timedelta_val): - from gcloud_bigtable._helpers import _timedelta_to_duration_pb - return _timedelta_to_duration_pb(timedelta_val) - - def test_it(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = microseconds = 1 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds) - self.assertEqual(result.nanos, 1000 * microseconds) - - def test_with_negative_microseconds(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = 1 - microseconds = -5 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds - 1) - self.assertEqual(result.nanos, 10**9 + 1000 * microseconds) - - def test_with_negative_seconds(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = -1 - microseconds = 5 - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(timedelta_val) - self.assertTrue(isinstance(result, duration_pb2.Duration)) - self.assertEqual(result.seconds, seconds + 1) - self.assertEqual(result.nanos, -(10**9 - 1000 * microseconds)) - - -class Test__duration_pb_to_timedelta(unittest2.TestCase): - - def _callFUT(self, duration_pb): - from gcloud_bigtable._helpers import _duration_pb_to_timedelta - return _duration_pb_to_timedelta(duration_pb) - - def test_it(self): - import datetime - from gcloud_bigtable._generated import duration_pb2 - - seconds = microseconds = 1 - duration_pb = duration_pb2.Duration(seconds=seconds, - nanos=1000 * microseconds) - timedelta_val = datetime.timedelta(seconds=seconds, - microseconds=microseconds) - result = self._callFUT(duration_pb) - self.assertTrue(isinstance(result, datetime.timedelta)) - self.assertEqual(result, timedelta_val) - - -class Test__timestamp_to_microseconds(unittest2.TestCase): - - def _callFUT(self, timestamp, granularity=1000): - from gcloud_bigtable._helpers import _timestamp_to_microseconds - return _timestamp_to_microseconds(timestamp, granularity=granularity) - - def test_default_granularity(self): - import datetime - from gcloud_bigtable import _helpers as MUT - - microseconds = 898294371 - millis_granularity = microseconds - (microseconds % 1000) - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(millis_granularity, self._callFUT(timestamp)) - - def test_no_granularity(self): - import datetime - from gcloud_bigtable import _helpers as MUT - - microseconds = 11122205067 - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(microseconds, self._callFUT(timestamp, granularity=1)) - - def test_non_utc_timestamp(self): - import datetime - - epoch_no_tz = datetime.datetime.utcfromtimestamp(0) - with self.assertRaises(TypeError): - self._callFUT(epoch_no_tz) - - def test_non_datetime_timestamp(self): - timestamp = object() # Not a datetime object. - with self.assertRaises(TypeError): - self._callFUT(timestamp) - - -class Test__microseconds_to_timestamp(unittest2.TestCase): - - def _callFUT(self, microseconds): - from gcloud_bigtable._helpers import _microseconds_to_timestamp - return _microseconds_to_timestamp(microseconds) - - def test_it(self): - import datetime - from gcloud_bigtable import _helpers as MUT - - microseconds = 123456 - timestamp = MUT.EPOCH + datetime.timedelta(microseconds=microseconds) - self.assertEqual(timestamp, self._callFUT(microseconds)) - - -class Test__set_certs(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._helpers import _set_certs - return _set_certs() - - def test_it(self): - import tempfile - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - self.assertTrue(MUT.AuthInfo.ROOT_CERTIFICATES is None) - - filename = tempfile.mktemp() - contents = b'FOOBARBAZ' - with open(filename, 'wb') as file_obj: - file_obj.write(contents) - with _Monkey(MUT, SSL_CERT_FILE=filename): - self._callFUT() - - self.assertEqual(MUT.AuthInfo.ROOT_CERTIFICATES, contents) - # Reset to `None` value checked above. - MUT.AuthInfo.ROOT_CERTIFICATES = None - - -class Test_set_certs(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._helpers import set_certs - return set_certs() - - def test_call_private(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - call_count = [0] - - def mock_set_certs(): - call_count[0] += 1 - - class _AuthInfo(object): - ROOT_CERTIFICATES = None - - with _Monkey(MUT, AuthInfo=_AuthInfo, - _set_certs=mock_set_certs): - self._callFUT() - - self.assertEqual(call_count, [1]) - - def test_do_nothing(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - call_count = [0] - - def mock_set_certs(): - call_count[0] += 1 - - # Make sure the fake method gets called by **someone** to make - # tox -e cover happy. - mock_set_certs() - self.assertEqual(call_count, [1]) - - class _AuthInfo(object): - ROOT_CERTIFICATES = object() - - with _Monkey(MUT, AuthInfo=_AuthInfo, - _set_certs=mock_set_certs): - self._callFUT() - - self.assertEqual(call_count, [1]) - - -class Test_get_certs(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable._helpers import get_certs - return get_certs() - - def test_it(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - call_kwargs = [] - return_val = object() - - def mock_set_certs(**kwargs): - call_kwargs.append(kwargs) - - class _AuthInfo(object): - ROOT_CERTIFICATES = return_val - - with _Monkey(MUT, AuthInfo=_AuthInfo, - set_certs=mock_set_certs): - result = self._callFUT() - - self.assertEqual(call_kwargs, [{'reset': False}]) - self.assertTrue(result is return_val) - - -class Test_make_stub(unittest2.TestCase): - - def _callFUT(self, credentials, stub_factory, host, port): - from gcloud_bigtable._helpers import make_stub - return make_stub(credentials, stub_factory, host, port) - - def test_it(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import _helpers as MUT - - mock_result = object() - custom_factory = _MockCalled(mock_result) - transformed = object() - transformer = _MockCalled(transformed) - - host = 'HOST' - port = 1025 - certs = 'FOOBAR' - client = _MockWithAttachedMethods() - with _Monkey(MUT, get_certs=lambda: certs, - MetadataTransformer=transformer): - result = self._callFUT(client, custom_factory, host, port) - - self.assertTrue(result is mock_result) - custom_factory.check_called( - self, - [(host, port)], - [{ - 'metadata_transformer': transformed, - 'secure': True, - 'root_certificates': certs, - }], - ) - transformer.check_called(self, [(client,)]) - self.assertEqual(client._called, []) - - -class Test__parse_family_pb(unittest2.TestCase): - - def _callFUT(self, family_pb): - from gcloud_bigtable._helpers import _parse_family_pb - return _parse_family_pb(family_pb) - - def test_it(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._helpers import _microseconds_to_timestamp - - COL_FAM1 = u'col-fam-id' - COL_NAME1 = b'col-name1' - COL_NAME2 = b'col-name2' - CELL_VAL1 = b'cell-val' - CELL_VAL2 = b'cell-val-newer' - CELL_VAL3 = b'altcol-cell-val' - - microseconds = 5554441037 - timestamp = _microseconds_to_timestamp(microseconds) - expected_dict = { - COL_NAME1: [ - (CELL_VAL1, timestamp), - (CELL_VAL2, timestamp), - ], - COL_NAME2: [ - (CELL_VAL3, timestamp), - ], - } - expected_output = (COL_FAM1, expected_dict) - sample_input = data_pb2.Family( - name=COL_FAM1, - columns=[ - data_pb2.Column( - qualifier=COL_NAME1, - cells=[ - data_pb2.Cell( - value=CELL_VAL1, - timestamp_micros=microseconds, - ), - data_pb2.Cell( - value=CELL_VAL2, - timestamp_micros=microseconds, - ), - ], - ), - data_pb2.Column( - qualifier=COL_NAME2, - cells=[ - data_pb2.Cell( - value=CELL_VAL3, - timestamp_micros=microseconds, - ), - ], - ), - ], - ) - self.assertEqual(expected_output, self._callFUT(sample_input)) - - -class Test__to_bytes(unittest2.TestCase): - - def _callFUT(self, *args, **kwargs): - from gcloud_bigtable._helpers import _to_bytes - return _to_bytes(*args, **kwargs) - - def test_with_bytes(self): - value = b'bytes-val' - self.assertEqual(self._callFUT(value), value) - - def test_with_unicode(self): - value = u'string-val' - encoded_value = b'string-val' - self.assertEqual(self._callFUT(value), encoded_value) - - def test_unicode_non_ascii(self): - value = u'\u2013' # Long hyphen - encoded_value = b'\xe2\x80\x93' - self.assertRaises(UnicodeEncodeError, self._callFUT, value) - self.assertEqual(self._callFUT(value, encoding='utf-8'), - encoded_value) - - def test_with_nonstring_type(self): - value = object() - self.assertRaises(TypeError, self._callFUT, value) diff --git a/gcloud_bigtable/test_client.py b/gcloud_bigtable/test_client.py deleted file mode 100644 index 324914c..0000000 --- a/gcloud_bigtable/test_client.py +++ /dev/null @@ -1,931 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT = 'project-id' - - -class Test__project_from_environment(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable.client import _project_from_environment - return _project_from_environment() - - def test_it(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - fake_project = object() - mock_os = _MockWithAttachedMethods(fake_project) - with _Monkey(MUT, os=mock_os): - result = self._callFUT() - - self.assertTrue(result is fake_project) - self.assertEqual(mock_os._called, - [('getenv', (MUT.PROJECT_ENV_VAR,), {})]) - - -class Test__project_from_app_engine(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable.client import _project_from_app_engine - return _project_from_app_engine() - - def test_without_app_engine(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - with _Monkey(MUT, app_identity=None): - result = self._callFUT() - - self.assertEqual(result, None) - - def test_with_app_engine(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - fake_project = object() - mock_app_identity = _MockWithAttachedMethods(fake_project) - with _Monkey(MUT, app_identity=mock_app_identity): - result = self._callFUT() - - self.assertTrue(result is fake_project) - self.assertEqual(mock_app_identity._called, - [('get_application_id', (), {})]) - - -class Test__project_from_compute_engine(unittest2.TestCase): - - def _callFUT(self): - from gcloud_bigtable.client import _project_from_compute_engine - return _project_from_compute_engine() - - @staticmethod - def _make_http_connection_response(status_code, read_result, - raise_socket_err=False): - import socket - - class Response(object): - status = status_code - - @staticmethod - def read(): - if raise_socket_err: - raise socket.error('Failed') - else: - return read_result - return Response - - @staticmethod - def _make_fake_six_module(mock_http_client): - class MockSix(object): - class moves(object): - http_client = mock_http_client - return MockSix - - def _helper(self, status, raise_socket_err=False): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - fake_project = object() - response = self._make_http_connection_response( - status, fake_project, raise_socket_err=raise_socket_err) - # The connection does the bulk of the work. - mock_connection = _MockWithAttachedMethods(None, response, None) - # The http_client module holds the connection constructor. - mock_http_client = _MockWithAttachedMethods(mock_connection) - # We need to put the client in place of it's location in six. - mock_six = self._make_fake_six_module(mock_http_client) - - with _Monkey(MUT, six=mock_six): - result = self._callFUT() - - if status == 200 and not raise_socket_err: - self.assertEqual(result, fake_project) - else: - self.assertEqual(result, None) - - self.assertEqual(mock_connection._called, [ - ( - 'request', - ('GET', '/computeMetadata/v1/project/project-id'), - {'headers': {'Metadata-Flavor': 'Google'}}, - ), - ( - 'getresponse', - (), - {}, - ), - ( - 'close', - (), - {}, - ), - ]) - self.assertEqual(mock_http_client._called, [ - ( - 'HTTPConnection', - ('169.254.169.254',), - {'timeout': 0.1}, - ), - ]) - - def test_success(self): - self._helper(200) - - def test_failed_status(self): - self._helper(404) - - def test_read_fails_with_socket_error(self): - self._helper(200, raise_socket_err=True) - - -class Test__determine_project(unittest2.TestCase): - - def _callFUT(self, project): - from gcloud_bigtable.client import _determine_project - return _determine_project(project) - - def _helper(self, num_mocks_called, mock_output, method_input): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - mock_project_from_environment = _MockCalled(None) - mock_project_from_app_engine = _MockCalled(None) - mock_project_from_compute_engine = _MockCalled(None) - - monkey_kwargs = { - '_project_from_environment': mock_project_from_environment, - '_project_from_app_engine': mock_project_from_app_engine, - '_project_from_compute_engine': ( - mock_project_from_compute_engine), - } - # Need the mocks in order they are called, so we can - # access them based on `num_mocks_called`. - mocks = [ - mock_project_from_environment, - mock_project_from_app_engine, - mock_project_from_compute_engine, - ] - mocks[num_mocks_called - 1].result = mock_output - - with _Monkey(MUT, **monkey_kwargs): - if num_mocks_called == 3 and mock_output is None: - with self.assertRaises(EnvironmentError): - self._callFUT(method_input) - else: - result = self._callFUT(method_input) - self.assertEqual(result, method_input or mock_output) - - # Make sure our mocks were called with no arguments. - for mock in mocks[:num_mocks_called]: - mock.check_called(self, [()]) - for mock in mocks[num_mocks_called:]: - mock.check_called(self, []) - - def test_fail_to_infer(self): - self._helper(num_mocks_called=3, mock_output=None, - method_input=None) - - def test_with_explicit_value(self): - self._helper(num_mocks_called=0, mock_output=None, - method_input=PROJECT) - - def test_from_environment(self): - self._helper(num_mocks_called=1, mock_output=PROJECT, - method_input=None) - - def test_from_app_engine(self): - self._helper(num_mocks_called=2, mock_output=PROJECT, - method_input=None) - - def test_from_compute_engine(self): - self._helper(num_mocks_called=3, mock_output=PROJECT, - method_input=None) - - -class TestClient(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.client import Client - return Client - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def _constructor_test_helper(self, expected_scopes, project=None, - read_only=False, admin=False, - user_agent=None): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - determined_project = object() - mock_determine_project = _MockCalled(determined_project) - with _Monkey(MUT, _determine_project=mock_determine_project): - client = self._makeOne(project=project, credentials=credentials, - read_only=read_only, admin=admin, - user_agent=user_agent) - - self.assertTrue(client._credentials is scoped_creds) - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - self.assertTrue(client.project is determined_project) - self.assertEqual(client.timeout_seconds, MUT.DEFAULT_TIMEOUT_SECONDS) - self.assertEqual(client.user_agent, user_agent) - mock_determine_project.check_called(self, [(project,)]) - - def test_constructor_default(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - mock_creds_class = _MockWithAttachedMethods(credentials) - - with _Monkey(MUT, GoogleCredentials=mock_creds_class): - client = self._makeOne(project=PROJECT) - - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is scoped_creds) - self.assertEqual(client.user_agent, MUT.DEFAULT_USER_AGENT) - self.assertEqual(mock_creds_class._called, - [('get_application_default', (), {})]) - expected_scopes = [MUT.DATA_SCOPE] - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - - def test_constructor_explicit_credentials(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.DATA_SCOPE] - self._constructor_test_helper(expected_scopes) - - def test_constructor_with_explicit_project(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.DATA_SCOPE] - self._constructor_test_helper(expected_scopes, project=PROJECT) - - def test_constructor_with_explicit_user_agent(self): - from gcloud_bigtable import client as MUT - user_agent = 'USER_AGENT' - expected_scopes = [MUT.DATA_SCOPE] - self._constructor_test_helper(expected_scopes, user_agent=user_agent) - - def test_constructor_with_admin(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.DATA_SCOPE, MUT.ADMIN_SCOPE] - self._constructor_test_helper(expected_scopes, admin=True) - - def test_constructor_with_read_only(self): - from gcloud_bigtable import client as MUT - expected_scopes = [MUT.READ_ONLY_SCOPE] - self._constructor_test_helper(expected_scopes, read_only=True) - - def test_constructor_both_admin_and_read_only(self): - with self.assertRaises(ValueError): - self._makeOne(project=PROJECT, credentials=None, - admin=True, read_only=True) - - def test_from_service_account_json(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - klass = self._getTargetClass() - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - get_adc = _MockCalled(credentials) - json_credentials_path = 'JSON_CREDENTIALS_PATH' - - with _Monkey(MUT, - _get_application_default_credential_from_file=get_adc): - client = klass.from_service_account_json( - json_credentials_path, project=PROJECT) - - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is scoped_creds) - - expected_scopes = [MUT.DATA_SCOPE] - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - # _get_application_default_credential_from_file only has pos. args. - get_adc.check_called(self, [(json_credentials_path,)]) - - def test_from_service_account_p12(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - klass = self._getTargetClass() - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - signed_creds = _MockCalled(credentials) - - private_key = 'PRIVATE_KEY' - mock_get_contents = _MockCalled(private_key) - client_email = 'CLIENT_EMAIL' - private_key_path = 'PRIVATE_KEY_PATH' - - with _Monkey(MUT, SignedJwtAssertionCredentials=signed_creds, - _get_contents=mock_get_contents): - client = klass.from_service_account_p12( - client_email, private_key_path, project=PROJECT) - - self.assertEqual(client.project, PROJECT) - self.assertTrue(client._credentials is scoped_creds) - expected_scopes = [MUT.DATA_SCOPE] - self.assertEqual(credentials._called, [ - ('create_scoped', (expected_scopes,), {}), - ]) - # SignedJwtAssertionCredentials() called with only kwargs - signed_creds_kw = { - 'private_key': private_key, - 'service_account_name': client_email, - } - signed_creds.check_called(self, [()], [signed_creds_kw]) - # Load private key (via _get_contents) from the key path. - mock_get_contents.check_called(self, [(private_key_path,)]) - - def _copy_test_helper(self, read_only=False, admin=False): - class Credentials(object): - - scopes = None - - def __init__(self, value=None): - self.value = value - - def create_scoped(self, scope): - self.scopes = scope - return self - - def __eq__(self, other): - return self.value == other.value - - credentials = Credentials('value') - timeout_seconds = 123 - user_agent = 'you-sir-age-int' - client = self._makeOne(project=PROJECT, credentials=credentials, - read_only=read_only, admin=admin, - timeout_seconds=timeout_seconds, - user_agent=user_agent) - # Put some fake stubs in place so that we can verify they - # don't get copied. - client._data_stub_internal = object() - client._cluster_stub_internal = object() - client._operations_stub_internal = object() - client._table_stub_internal = object() - - new_client = client.copy() - self.assertEqual(new_client._admin, client._admin) - self.assertEqual(new_client._credentials, client._credentials) - # Make sure credentials (a non-simple type) gets copied - # to a new instance. - self.assertFalse(new_client._credentials is client._credentials) - self.assertEqual(new_client.project, client.project) - self.assertEqual(new_client.user_agent, client.user_agent) - self.assertEqual(new_client.timeout_seconds, client.timeout_seconds) - # Make sure stubs are not preserved. - self.assertEqual(new_client._data_stub_internal, None) - self.assertEqual(new_client._cluster_stub_internal, None) - self.assertEqual(new_client._operations_stub_internal, None) - self.assertEqual(new_client._table_stub_internal, None) - - def test_copy(self): - self._copy_test_helper() - - def test_copy_admin(self): - self._copy_test_helper(admin=True) - - def test_copy_read_only(self): - self._copy_test_helper(read_only=True) - - def test_credentials_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertTrue(client.credentials is scoped_creds) - - def test_project_name_property(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - project_name = 'projects/' + PROJECT - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertEqual(client.project_name, project_name) - - def test_data_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - client._data_stub_internal = object() - self.assertTrue(client._data_stub is client._data_stub_internal) - - def test_data_stub_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - with self.assertRaises(ValueError): - getattr(client, '_data_stub') - - def test_cluster_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - client._cluster_stub_internal = object() - self.assertTrue(client._cluster_stub is client._cluster_stub_internal) - - def test_cluster_stub_non_admin_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=False) - with self.assertRaises(ValueError): - getattr(client, '_cluster_stub') - - def test_cluster_stub_unset_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - with self.assertRaises(ValueError): - getattr(client, '_cluster_stub') - - def test_operations_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - client._operations_stub_internal = object() - self.assertTrue(client._operations_stub is - client._operations_stub_internal) - - def test_operations_stub_non_admin_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=False) - with self.assertRaises(ValueError): - getattr(client, '_operations_stub') - - def test_operations_stub_unset_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - with self.assertRaises(ValueError): - getattr(client, '_operations_stub') - - def test_table_stub_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - client._table_stub_internal = object() - self.assertTrue(client._table_stub is client._table_stub_internal) - - def test_table_stub_non_admin_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=False) - with self.assertRaises(ValueError): - getattr(client, '_table_stub') - - def test_table_stub_unset_failure(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - with self.assertRaises(ValueError): - getattr(client, '_table_stub') - - def test__make_data_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import DATA_API_HOST - from gcloud_bigtable.client import DATA_API_PORT - from gcloud_bigtable.client import DATA_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_data_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - DATA_STUB_FACTORY, - DATA_API_HOST, - DATA_API_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test__make_cluster_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import CLUSTER_ADMIN_HOST - from gcloud_bigtable.client import CLUSTER_ADMIN_PORT - from gcloud_bigtable.client import CLUSTER_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_cluster_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - CLUSTER_STUB_FACTORY, - CLUSTER_ADMIN_HOST, - CLUSTER_ADMIN_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test__make_operations_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import CLUSTER_ADMIN_HOST - from gcloud_bigtable.client import CLUSTER_ADMIN_PORT - from gcloud_bigtable.client import OPERATIONS_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_operations_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - OPERATIONS_STUB_FACTORY, - CLUSTER_ADMIN_HOST, - CLUSTER_ADMIN_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test__make_table_stub(self): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - from gcloud_bigtable.client import TABLE_ADMIN_HOST - from gcloud_bigtable.client import TABLE_ADMIN_PORT - from gcloud_bigtable.client import TABLE_STUB_FACTORY - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - expected_result = object() - mock_make_stub = _MockCalled(expected_result) - with _Monkey(MUT, make_stub=mock_make_stub): - result = client._make_table_stub() - - self.assertTrue(result is expected_result) - make_stub_args = [ - ( - client, - TABLE_STUB_FACTORY, - TABLE_ADMIN_HOST, - TABLE_ADMIN_PORT, - ), - ] - mock_make_stub.check_called(self, make_stub_args) - - def test_is_started(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertFalse(client.is_started()) - client._data_stub_internal = object() - self.assertTrue(client.is_started()) - client._data_stub_internal = None - self.assertFalse(client.is_started()) - - def _start_method_helper(self, admin): - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import client as MUT - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=admin) - stub = _FakeStub() - mock_make_stub = _MockCalled(stub) - with _Monkey(MUT, make_stub=mock_make_stub): - client.start() - - self.assertTrue(client._data_stub_internal is stub) - if admin: - self.assertTrue(client._cluster_stub_internal is stub) - self.assertTrue(client._operations_stub_internal is stub) - self.assertTrue(client._table_stub_internal is stub) - self.assertEqual(stub._entered, 4) - else: - self.assertTrue(client._cluster_stub_internal is None) - self.assertTrue(client._operations_stub_internal is None) - self.assertTrue(client._table_stub_internal is None) - self.assertEqual(stub._entered, 1) - self.assertEqual(stub._exited, []) - - def test_start_non_admin(self): - self._start_method_helper(admin=False) - - def test_start_with_admin(self): - self._start_method_helper(admin=True) - - def test_start_while_started(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - client._data_stub_internal = data_stub = object() - self.assertTrue(client.is_started()) - client.start() - - # Make sure the stub did not change. - self.assertEqual(client._data_stub_internal, data_stub) - - def _stop_method_helper(self, admin): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=admin) - stub1 = _FakeStub() - stub2 = _FakeStub() - client._data_stub_internal = stub1 - client._cluster_stub_internal = stub2 - client._operations_stub_internal = stub2 - client._table_stub_internal = stub2 - client.stop() - self.assertTrue(client._data_stub_internal is None) - self.assertTrue(client._cluster_stub_internal is None) - self.assertTrue(client._operations_stub_internal is None) - self.assertTrue(client._table_stub_internal is None) - self.assertEqual(stub1._entered, 0) - self.assertEqual(stub2._entered, 0) - exc_none_triple = (None, None, None) - self.assertEqual(stub1._exited, [exc_none_triple]) - if admin: - self.assertEqual(stub2._exited, [exc_none_triple] * 3) - else: - self.assertEqual(stub2._exited, []) - - def test_stop_non_admin(self): - self._stop_method_helper(admin=False) - - def test_stop_with_admin(self): - self._stop_method_helper(admin=True) - - def test_stop_while_stopped(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - self.assertFalse(client.is_started()) - - # This is a bit hacky. We set the cluster stub protected value - # since it isn't used in is_started() and make sure that stop - # doesn't reset this value to None. - client._cluster_stub_internal = cluster_stub = object() - client.stop() - # Make sure the cluster stub did not change. - self.assertEqual(client._cluster_stub_internal, cluster_stub) - - def test_cluster_factory(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.cluster import Cluster - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, credentials=credentials) - - zone = 'zone' - cluster_id = 'cluster-id' - cluster = client.cluster(zone, cluster_id) - self.assertTrue(isinstance(cluster, Cluster)) - self.assertTrue(cluster.client is client) - self.assertEqual(cluster.zone, zone) - self.assertEqual(cluster.cluster_id, cluster_id) - - def _list_zones_helper(self, zone_status): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - - # Create request_pb - request_pb = messages_pb2.ListZonesRequest( - name='projects/' + PROJECT, - ) - - # Create response_pb - zone1 = 'foo' - zone2 = 'bar' - response_pb = messages_pb2.ListZonesResponse( - zones=[ - data_pb2.Zone(display_name=zone1, status=zone_status), - data_pb2.Zone(display_name=zone2, status=zone_status), - ], - ) - - # Patch the stub used by the API method. - client._cluster_stub_internal = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = [zone1, zone2] - - # Perform the method and check the result. - timeout_seconds = 281330 - result = client.list_zones(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ListZones', - (request_pb, timeout_seconds), - {}, - )]) - - def test_list_zones(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - self._list_zones_helper(data_pb2.Zone.OK) - - def test_list_zones_failure(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - with self.assertRaises(ValueError): - self._list_zones_helper(data_pb2.Zone.EMERGENCY_MAINENANCE) - - def test_list_clusters(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockWithAttachedMethods - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = self._makeOne(project=PROJECT, - credentials=credentials, admin=True) - - # Create request_pb - request_pb = messages_pb2.ListClustersRequest( - name='projects/' + PROJECT, - ) - - # Create response_pb - zone = 'foo' - failed_zone = 'bar' - cluster_id1 = 'cluster-id1' - cluster_id2 = 'cluster-id2' - cluster_name1 = ('projects/' + PROJECT + '/zones/' + zone + - '/clusters/' + cluster_id1) - cluster_name2 = ('projects/' + PROJECT + '/zones/' + zone + - '/clusters/' + cluster_id2) - response_pb = messages_pb2.ListClustersResponse( - failed_zones=[ - data_pb2.Zone(display_name=failed_zone), - ], - clusters=[ - data_pb2.Cluster( - name=cluster_name1, - display_name=cluster_name1, - serve_nodes=3, - ), - data_pb2.Cluster( - name=cluster_name2, - display_name=cluster_name2, - serve_nodes=3, - ), - ], - ) - - # Patch the stub used by the API method. - client._cluster_stub_internal = stub = StubMock(response_pb) - - # Create expected_result. - failed_zones = [failed_zone] - clusters = [ - client.cluster(zone, cluster_id1), - client.cluster(zone, cluster_id2), - ] - expected_result = (clusters, failed_zones) - - # Perform the method and check the result. - timeout_seconds = 8004 - result = client.list_clusters(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ListClusters', - (request_pb, timeout_seconds), - {}, - )]) - - -class Test__get_contents(unittest2.TestCase): - - def _callFUT(self, filename): - from gcloud_bigtable.client import _get_contents - return _get_contents(filename) - - def test_it(self): - import tempfile - - filename = tempfile.mktemp() - contents = b'foobar' - with open(filename, 'wb') as file_obj: - file_obj.write(contents) - - self.assertEqual(self._callFUT(filename), contents) - - -class _FakeStub(object): - - def __init__(self): - self._entered = 0 - self._exited = [] - - def __enter__(self): - self._entered += 1 - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self._exited.append((exc_type, exc_val, exc_tb)) - return True diff --git a/gcloud_bigtable/test_cluster.py b/gcloud_bigtable/test_cluster.py deleted file mode 100644 index cbdf373..0000000 --- a/gcloud_bigtable/test_cluster.py +++ /dev/null @@ -1,627 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT = 'project-id' -ZONE = 'zone' -CLUSTER_ID = 'cluster-id' - - -class Test__prepare_create_request(unittest2.TestCase): - - def _callFUT(self, cluster): - from gcloud_bigtable.cluster import _prepare_create_request - return _prepare_create_request(cluster) - - def test_it(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.cluster import Cluster - display_name = 'DISPLAY_NAME' - serve_nodes = 8 - - cluster = Cluster(ZONE, CLUSTER_ID, _Client(PROJECT), - display_name=display_name, serve_nodes=serve_nodes) - request_pb = self._callFUT(cluster) - self.assertTrue(isinstance(request_pb, - messages_pb2.CreateClusterRequest)) - self.assertEqual(request_pb.cluster_id, CLUSTER_ID) - self.assertEqual(request_pb.name, - 'projects/' + PROJECT + '/zones/' + ZONE) - self.assertTrue(isinstance(request_pb.cluster, data_pb2.Cluster)) - self.assertEqual(request_pb.cluster.display_name, display_name) - self.assertEqual(request_pb.cluster.serve_nodes, serve_nodes) - - -class Test__process_operation(unittest2.TestCase): - - def _callFUT(self, operation_pb): - from gcloud_bigtable.cluster import _process_operation - return _process_operation(operation_pb) - - def test_it(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import operations_pb2 - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - expected_operation_id = 234 - operation_name = ('operations/projects/%s/zones/%s/clusters/%s/' - 'operations/%d' % (PROJECT, ZONE, CLUSTER_ID, - expected_operation_id)) - - current_op = operations_pb2.Operation(name=operation_name) - - request_metadata = messages_pb2.CreateClusterMetadata() - mock_parse_pb_any_to_native = _MockCalled(request_metadata) - expected_operation_begin = object() - mock_pb_timestamp_to_datetime = _MockCalled(expected_operation_begin) - with _Monkey(MUT, _parse_pb_any_to_native=mock_parse_pb_any_to_native, - _pb_timestamp_to_datetime=mock_pb_timestamp_to_datetime): - operation_id, operation_begin = self._callFUT(current_op) - - self.assertEqual(operation_id, expected_operation_id) - self.assertTrue(operation_begin is expected_operation_begin) - - mock_parse_pb_any_to_native.check_called( - self, [(current_op.metadata,)]) - mock_pb_timestamp_to_datetime.check_called( - self, [(request_metadata.request_time,)]) - - def test_failure(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - - cluster = data_pb2.Cluster() - with self.assertRaises(ValueError): - self._callFUT(cluster) - - -class TestCluster(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.cluster import Cluster - return Cluster - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - client = object() - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertEqual(cluster.zone, ZONE) - self.assertEqual(cluster.cluster_id, CLUSTER_ID) - self.assertEqual(cluster.display_name, CLUSTER_ID) - self.assertEqual(cluster.serve_nodes, 3) - self.assertTrue(cluster._client is client) - - def test_constructor_non_default(self): - client = object() - display_name = 'display_name' - serve_nodes = 8 - cluster = self._makeOne(ZONE, CLUSTER_ID, client, - display_name=display_name, - serve_nodes=serve_nodes) - self.assertEqual(cluster.zone, ZONE) - self.assertEqual(cluster.cluster_id, CLUSTER_ID) - self.assertEqual(cluster.display_name, display_name) - self.assertEqual(cluster.serve_nodes, serve_nodes) - self.assertTrue(cluster._client is client) - - def test_copy(self): - client = _Client(PROJECT) - display_name = 'DISPLAY_NAME' - serve_nodes = 8 - cluster = self._makeOne(ZONE, CLUSTER_ID, client, - display_name=display_name, - serve_nodes=serve_nodes) - new_cluster = cluster.copy() - - # Make sure the client got copied to a new instance. - self.assertFalse(new_cluster._client is client) - self.assertEqual(new_cluster._client.__dict__, - client.__dict__) - # Just replace the client on the new_cluster so we can - # check cluster equality. - new_cluster._client = client - self.assertFalse(cluster is new_cluster) - self.assertEqual(cluster, new_cluster) - - def test_client_getter(self): - client = object() - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertTrue(cluster.client is client) - - def test_project_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = Client(project=PROJECT, credentials=credentials) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertEqual(cluster.project, PROJECT) - - def test_timeout_seconds_getter(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - timeout_seconds = 77 - client = Client(project=PROJECT, credentials=credentials, - timeout_seconds=timeout_seconds) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - self.assertEqual(cluster.timeout_seconds, timeout_seconds) - - def test_name_property(self): - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = Client(project=PROJECT, credentials=credentials) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - self.assertEqual(cluster.name, cluster_name) - - def test_table_factory(self): - from gcloud_bigtable.table import Table - - cluster = self._makeOne(ZONE, CLUSTER_ID, None) - table_id = 'table_id' - table = cluster.table(table_id) - self.assertTrue(isinstance(table, Table)) - self.assertEqual(table.table_id, table_id) - self.assertEqual(table._cluster, cluster) - - def test_from_pb_success(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - client = Client(project=PROJECT, credentials=credentials) - - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster_pb = data_pb2.Cluster( - name=cluster_name, - display_name=CLUSTER_ID, - serve_nodes=3, - ) - - klass = self._getTargetClass() - cluster = klass.from_pb(cluster_pb, client) - self.assertTrue(isinstance(cluster, klass)) - self.assertEqual(cluster.client, client) - self.assertEqual(cluster.zone, ZONE) - self.assertEqual(cluster.cluster_id, CLUSTER_ID) - - def test_from_pb_bad_cluster_name(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - - cluster_name = 'INCORRECT_FORMAT' - cluster_pb = data_pb2.Cluster(name=cluster_name) - - klass = self._getTargetClass() - with self.assertRaises(ValueError): - klass.from_pb(cluster_pb, None) - - def test_from_pb_project_mistmatch(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._testing import _MockWithAttachedMethods - from gcloud_bigtable.client import Client - - scoped_creds = object() - credentials = _MockWithAttachedMethods(scoped_creds) - alt_project = 'ALT_PROJECT' - client = Client(project=alt_project, credentials=credentials) - - self.assertNotEqual(PROJECT, alt_project) - - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster_pb = data_pb2.Cluster(name=cluster_name) - - klass = self._getTargetClass() - with self.assertRaises(ValueError): - klass.from_pb(cluster_pb, client) - - def test___eq__(self): - zone = 'zone' - cluster_id = 'cluster_id' - client = object() - cluster1 = self._makeOne(zone, cluster_id, client) - cluster2 = self._makeOne(zone, cluster_id, client) - self.assertEqual(cluster1, cluster2) - - def test___eq__type_differ(self): - cluster1 = self._makeOne('zone', 'cluster_id', 'client') - cluster2 = object() - self.assertNotEqual(cluster1, cluster2) - - def test___ne__same_value(self): - zone = 'zone' - cluster_id = 'cluster_id' - client = object() - cluster1 = self._makeOne(zone, cluster_id, client) - cluster2 = self._makeOne(zone, cluster_id, client) - comparison_val = (cluster1 != cluster2) - self.assertFalse(comparison_val) - - def test___ne__(self): - cluster1 = self._makeOne('zone1', 'cluster_id1', 'client1') - cluster2 = self._makeOne('zone2', 'cluster_id2', 'client2') - self.assertNotEqual(cluster1, cluster2) - - def test_reload(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client(PROJECT) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = messages_pb2.GetClusterRequest(name=cluster_name) - - # Create response_pb - response_pb = data_pb2.Cluster( - display_name=CLUSTER_ID, - serve_nodes=3, - ) - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # reload() has no return value. - - # Perform the method and check the result. - timeout_seconds = 123 - result = cluster.reload(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'GetCluster', - (request_pb, timeout_seconds), - {}, - )]) - - def test_operation_finished_without_operation(self): - cluster = self._makeOne(ZONE, CLUSTER_ID, None) - self.assertEqual(cluster._operation_type, None) - with self.assertRaises(ValueError): - cluster.operation_finished() - - def _operation_finished_helper(self, done): - from gcloud_bigtable._generated import operations_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client(PROJECT) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Patch up the cluster's operation attributes. - cluster._operation_id = op_id = 789 - cluster._operation_begin = op_begin = object() - cluster._operation_type = op_type = object() - - # Create request_pb - op_name = ('operations/projects/' + PROJECT + '/zones/' + - ZONE + '/clusters/' + CLUSTER_ID + - '/operations/%d' % (op_id,)) - request_pb = operations_pb2.GetOperationRequest(name=op_name) - - # Create response_pb - response_pb = operations_pb2.Operation(done=done) - - # Patch the stub used by the API method. - client._operations_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = done - - # Perform the method and check the result. - timeout_seconds = 1 - result = cluster.operation_finished(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'GetOperation', - (request_pb, timeout_seconds), - {}, - )]) - - if done: - self.assertEqual(cluster._operation_type, None) - self.assertEqual(cluster._operation_id, None) - self.assertEqual(cluster._operation_begin, None) - else: - self.assertEqual(cluster._operation_type, op_type) - self.assertEqual(cluster._operation_id, op_id) - self.assertEqual(cluster._operation_begin, op_begin) - - def test_operation_finished(self): - self._operation_finished_helper(done=True) - - def test_operation_finished_not_done(self): - self._operation_finished_helper(done=False) - - def test_create(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import operations_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - client = _Client(PROJECT) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb. Just a mock since we monkey patch - # _prepare_create_request - request_pb = object() - - # Create response_pb - current_op = operations_pb2.Operation() - response_pb = data_pb2.Cluster(current_operation=current_op) - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None - - # Perform the method and check the result. - timeout_seconds = 578 - mock_prepare_create_request = _MockCalled(request_pb) - op_id = 5678 - op_begin = object() - mock_process_operation = _MockCalled((op_id, op_begin)) - with _Monkey(MUT, _prepare_create_request=mock_prepare_create_request, - _process_operation=mock_process_operation): - result = cluster.create(timeout_seconds=timeout_seconds) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CreateCluster', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(cluster._operation_type, 'create') - self.assertEqual(cluster._operation_id, op_id) - self.assertTrue(cluster._operation_begin is op_begin) - mock_prepare_create_request.check_called(self, [(cluster,)]) - mock_process_operation.check_called(self, [(current_op,)]) - - def test_update(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_data_pb2 as data_pb2) - from gcloud_bigtable._generated import operations_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - client = _Client(PROJECT) - serve_nodes = 81 - display_name = 'display_name' - cluster = self._makeOne(ZONE, CLUSTER_ID, client, - display_name=display_name, - serve_nodes=serve_nodes) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = data_pb2.Cluster( - name=cluster_name, - display_name=display_name, - serve_nodes=serve_nodes, - ) - - # Create response_pb - current_op = operations_pb2.Operation() - response_pb = data_pb2.Cluster(current_operation=current_op) - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None - - # We must create the cluster object with the client passed in. - timeout_seconds = 9 - op_id = 5678 - op_begin = object() - mock_process_operation = _MockCalled((op_id, op_begin)) - with _Monkey(MUT, - _process_operation=mock_process_operation): - result = cluster.update(timeout_seconds=timeout_seconds) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'UpdateCluster', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(cluster._operation_type, 'update') - self.assertEqual(cluster._operation_id, op_id) - self.assertTrue(cluster._operation_begin is op_begin) - mock_process_operation.check_called(self, [(current_op,)]) - - def test_delete(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client(PROJECT) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = messages_pb2.DeleteClusterRequest(name=cluster_name) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # delete() has no return value. - - # Perform the method and check the result. - timeout_seconds = 57 - result = cluster.delete(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'DeleteCluster', - (request_pb, timeout_seconds), - {}, - )]) - - def test_undelete(self): - from gcloud_bigtable._generated import ( - bigtable_cluster_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import operations_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import cluster as MUT - - client = _Client(PROJECT) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_pb - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = messages_pb2.UndeleteClusterRequest(name=cluster_name) - - # Create response_pb - response_pb = operations_pb2.Operation() - - # Patch the stub used by the API method. - client._cluster_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None - - # Perform the method and check the result. - timeout_seconds = 78 - op_id = 5678 - op_begin = object() - mock_process_operation = _MockCalled((op_id, op_begin)) - with _Monkey(MUT, - _process_operation=mock_process_operation): - result = cluster.undelete(timeout_seconds=timeout_seconds) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'UndeleteCluster', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(cluster._operation_type, 'undelete') - self.assertEqual(cluster._operation_id, op_id) - self.assertTrue(cluster._operation_begin is op_begin) - mock_process_operation.check_called(self, [(response_pb,)]) - - def _list_tables_helper(self, table_id, table_name=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as table_data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as table_messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client(PROJECT) - cluster = self._makeOne(ZONE, CLUSTER_ID, client) - - # Create request_ - cluster_name = ('projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - request_pb = table_messages_pb2.ListTablesRequest(name=cluster_name) - - # Create response_pb - table_name = table_name or (cluster_name + '/tables/' + table_id) - response_pb = table_messages_pb2.ListTablesResponse( - tables=[ - table_data_pb2.Table(name=table_name), - ], - ) - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_table = cluster.table(table_id) - expected_result = [expected_table] - - # Perform the method and check the result. - timeout_seconds = 45 - result = cluster.list_tables(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ListTables', - (request_pb, timeout_seconds), - {}, - )]) - - def test_list_tables(self): - table_id = 'table_id' - self._list_tables_helper(table_id) - - def test_list_tables_failure_bad_split(self): - with self.assertRaises(ValueError): - self._list_tables_helper(None, table_name='wrong-format') - - def test_list_tables_failure_name_bad_before(self): - table_id = 'table_id' - bad_table_name = ('nonempty-section-before' + - 'projects/' + PROJECT + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + table_id) - with self.assertRaises(ValueError): - self._list_tables_helper(table_id, table_name=bad_table_name) - - -class _Client(object): - - cluster_stub = None - operations_stub = None - table_stub = None - - def __init__(self, project): - self.project = project - self.project_name = 'projects/' + project - - def copy(self): - import copy as copy_module - return copy_module.deepcopy(self) diff --git a/gcloud_bigtable/test_column_family.py b/gcloud_bigtable/test_column_family.py deleted file mode 100644 index 66e7282..0000000 --- a/gcloud_bigtable/test_column_family.py +++ /dev/null @@ -1,582 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT_ID = 'project-id' -ZONE = 'zone' -CLUSTER_ID = 'cluster-id' -TABLE_ID = 'table-id' -COLUMN_FAMILY_ID = 'column-family-id' - - -class TestGarbageCollectionRule(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - return GarbageCollectionRule - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - gc_rule = self._makeOne() - self.assertEqual(gc_rule.max_num_versions, None) - self.assertEqual(gc_rule.max_age, None) - - def test_constructor_failure(self): - with self.assertRaises(TypeError): - self._makeOne(max_num_versions=1, max_age=object()) - - def test___eq__max_age(self): - max_age = object() - gc_rule1 = self._makeOne(max_age=max_age) - gc_rule2 = self._makeOne(max_age=max_age) - self.assertEqual(gc_rule1, gc_rule2) - - def test___eq__max_num_versions(self): - gc_rule1 = self._makeOne(max_num_versions=2) - gc_rule2 = self._makeOne(max_num_versions=2) - self.assertEqual(gc_rule1, gc_rule2) - - def test___eq__type_differ(self): - gc_rule1 = self._makeOne() - gc_rule2 = object() - self.assertNotEqual(gc_rule1, gc_rule2) - - def test___ne__same_value(self): - gc_rule1 = self._makeOne() - gc_rule2 = self._makeOne() - comparison_val = (gc_rule1 != gc_rule2) - self.assertFalse(comparison_val) - - def test_to_pb_too_many_values(self): - # Fool the constructor by passing no values. - gc_rule = self._makeOne() - gc_rule.max_num_versions = object() - gc_rule.max_age = object() - with self.assertRaises(TypeError): - gc_rule.to_pb() - - def test_to_pb_no_value(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - gc_rule = self._makeOne() - pb_val = gc_rule.to_pb() - self.assertEqual(pb_val, data_pb2.GcRule()) - - def test_to_pb_with_max_num_versions(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - max_num_versions = 1337 - gc_rule = self._makeOne(max_num_versions=max_num_versions) - pb_val = gc_rule.to_pb() - self.assertEqual(pb_val, - data_pb2.GcRule(max_num_versions=max_num_versions)) - - def test_to_pb_with_max_age(self): - import datetime - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 - - max_age = datetime.timedelta(seconds=1) - duration = duration_pb2.Duration(seconds=1) - gc_rule = self._makeOne(max_age=max_age) - pb_val = gc_rule.to_pb() - self.assertEqual(pb_val, data_pb2.GcRule(max_age=duration)) - - -class TestGarbageCollectionRuleUnion(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import GarbageCollectionRuleUnion - return GarbageCollectionRuleUnion - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - rules = object() - rule_union = self._makeOne(rules=rules) - self.assertTrue(rule_union.rules is rules) - - def test___eq__(self): - rules = object() - gc_rule1 = self._makeOne(rules=rules) - gc_rule2 = self._makeOne(rules=rules) - self.assertEqual(gc_rule1, gc_rule2) - - def test___eq__type_differ(self): - gc_rule1 = self._makeOne() - gc_rule2 = object() - self.assertNotEqual(gc_rule1, gc_rule2) - - def test___ne__same_value(self): - gc_rule1 = self._makeOne() - gc_rule2 = self._makeOne() - comparison_val = (gc_rule1 != gc_rule2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - import datetime - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule - - max_num_versions = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions) - - max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - union=data_pb2.GcRule.Union(rules=[pb_rule1, pb_rule2])) - - gc_rule_pb = rule3.to_pb() - self.assertEqual(gc_rule_pb, pb_rule3) - - def test_to_pb_nested(self): - import datetime - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule - - max_num_versions1 = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions1) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions1) - - max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - union=data_pb2.GcRule.Union(rules=[pb_rule1, pb_rule2])) - - max_num_versions2 = 1337 - rule4 = GarbageCollectionRule(max_num_versions=max_num_versions2) - pb_rule4 = data_pb2.GcRule(max_num_versions=max_num_versions2) - - rule5 = self._makeOne(rules=[rule3, rule4]) - pb_rule5 = data_pb2.GcRule( - union=data_pb2.GcRule.Union(rules=[pb_rule3, pb_rule4])) - - gc_rule_pb = rule5.to_pb() - self.assertEqual(gc_rule_pb, pb_rule5) - - -class TestGarbageCollectionRuleIntersection(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - return GarbageCollectionRuleIntersection - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - rules = object() - rule_intersection = self._makeOne(rules=rules) - self.assertTrue(rule_intersection.rules is rules) - - def test___eq__(self): - rules = object() - gc_rule1 = self._makeOne(rules=rules) - gc_rule2 = self._makeOne(rules=rules) - self.assertEqual(gc_rule1, gc_rule2) - - def test___eq__type_differ(self): - gc_rule1 = self._makeOne() - gc_rule2 = object() - self.assertNotEqual(gc_rule1, gc_rule2) - - def test___ne__same_value(self): - gc_rule1 = self._makeOne() - gc_rule2 = self._makeOne() - comparison_val = (gc_rule1 != gc_rule2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - import datetime - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule - - max_num_versions = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions) - - max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - intersection=data_pb2.GcRule.Intersection( - rules=[pb_rule1, pb_rule2])) - - gc_rule_pb = rule3.to_pb() - self.assertEqual(gc_rule_pb, pb_rule3) - - def test_to_pb_nested(self): - import datetime - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 - from gcloud_bigtable.column_family import GarbageCollectionRule - - max_num_versions1 = 42 - rule1 = GarbageCollectionRule(max_num_versions=max_num_versions1) - pb_rule1 = data_pb2.GcRule(max_num_versions=max_num_versions1) - - max_age = datetime.timedelta(seconds=1) - rule2 = GarbageCollectionRule(max_age=max_age) - pb_rule2 = data_pb2.GcRule(max_age=duration_pb2.Duration(seconds=1)) - - rule3 = self._makeOne(rules=[rule1, rule2]) - pb_rule3 = data_pb2.GcRule( - intersection=data_pb2.GcRule.Intersection( - rules=[pb_rule1, pb_rule2])) - - max_num_versions2 = 1337 - rule4 = GarbageCollectionRule(max_num_versions=max_num_versions2) - pb_rule4 = data_pb2.GcRule(max_num_versions=max_num_versions2) - - rule5 = self._makeOne(rules=[rule3, rule4]) - pb_rule5 = data_pb2.GcRule( - intersection=data_pb2.GcRule.Intersection( - rules=[pb_rule3, pb_rule4])) - - gc_rule_pb = rule5.to_pb() - self.assertEqual(gc_rule_pb, pb_rule5) - - -class TestColumnFamily(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.column_family import ColumnFamily - return ColumnFamily - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - table = object() - gc_rule = object() - column_family = self._makeOne(COLUMN_FAMILY_ID, table, gc_rule=gc_rule) - self.assertEqual(column_family.column_family_id, COLUMN_FAMILY_ID) - self.assertTrue(column_family._table is table) - self.assertTrue(column_family.gc_rule is gc_rule) - - def test_table_getter(self): - table = object() - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - self.assertTrue(column_family.table is table) - - def test_client_getter(self): - client = object() - table = _Table(None, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - self.assertTrue(column_family.client is client) - - def test_timeout_seconds_getter(self): - timeout_seconds = 889 - table = _Table(None, timeout_seconds=timeout_seconds) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - self.assertEqual(column_family.timeout_seconds, timeout_seconds) - - def test_name_property(self): - table_name = 'table_name' - table = _Table(table_name) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - expected_name = table_name + '/columnFamilies/' + COLUMN_FAMILY_ID - self.assertEqual(column_family.name, expected_name) - - def test___eq__(self): - column_family_id = 'column_family_id' - table = object() - column_family1 = self._makeOne(column_family_id, table) - column_family2 = self._makeOne(column_family_id, table) - self.assertEqual(column_family1, column_family2) - - def test___eq__type_differ(self): - column_family1 = self._makeOne('column_family_id', None) - column_family2 = object() - self.assertNotEqual(column_family1, column_family2) - - def test___ne__same_value(self): - column_family_id = 'column_family_id' - table = object() - column_family1 = self._makeOne(column_family_id, table) - column_family2 = self._makeOne(column_family_id, table) - comparison_val = (column_family1 != column_family2) - self.assertFalse(comparison_val) - - def test___ne__(self): - column_family1 = self._makeOne('column_family_id1', 'table1') - column_family2 = self._makeOne('column_family_id2', 'table2') - self.assertNotEqual(column_family1, column_family2) - - def _create_test_helper(self, gc_rule=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) - table = _Table(table_name, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table, gc_rule=gc_rule) - - # Create request_pb - if gc_rule is None: - column_family_pb = data_pb2.ColumnFamily() - else: - column_family_pb = data_pb2.ColumnFamily(gc_rule=gc_rule.to_pb()) - request_pb = messages_pb2.CreateColumnFamilyRequest( - name=table_name, - column_family_id=COLUMN_FAMILY_ID, - column_family=column_family_pb, - ) - - # Create response_pb - response_pb = data_pb2.ColumnFamily() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # create() has no return value. - - # Perform the method and check the result. - timeout_seconds = 4 - result = column_family.create(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CreateColumnFamily', - (request_pb, timeout_seconds), - {}, - )]) - - def test_create(self): - self._create_test_helper(gc_rule=None) - - def test_create_with_gc_rule(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - gc_rule = GarbageCollectionRule(max_num_versions=1337) - self._create_test_helper(gc_rule=gc_rule) - - def _update_test_helper(self, gc_rule=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) - table = _Table(table_name, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table, gc_rule=gc_rule) - - # Create request_pb - column_family_name = table_name + '/columnFamilies/' + COLUMN_FAMILY_ID - if gc_rule is None: - request_pb = data_pb2.ColumnFamily(name=column_family_name) - else: - request_pb = data_pb2.ColumnFamily( - name=column_family_name, - gc_rule=gc_rule.to_pb(), - ) - - # Create response_pb - response_pb = data_pb2.ColumnFamily() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # update() has no return value. - - # Perform the method and check the result. - timeout_seconds = 28 - result = column_family.update(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'UpdateColumnFamily', - (request_pb, timeout_seconds), - {}, - )]) - - def test_update(self): - self._update_test_helper(gc_rule=None) - - def test_update_with_gc_rule(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - gc_rule = GarbageCollectionRule(max_num_versions=1337) - self._update_test_helper(gc_rule=gc_rule) - - def test_delete(self): - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) - table = _Table(table_name, client=client) - column_family = self._makeOne(COLUMN_FAMILY_ID, table) - - # Create request_pb - column_family_name = table_name + '/columnFamilies/' + COLUMN_FAMILY_ID - request_pb = messages_pb2.DeleteColumnFamilyRequest( - name=column_family_name) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # delete() has no return value. - - # Perform the method and check the result. - timeout_seconds = 7 - result = column_family.delete(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'DeleteColumnFamily', - (request_pb, timeout_seconds), - {}, - )]) - - -class Test__gc_rule_from_pb(unittest2.TestCase): - - def _callFUT(self, gc_rule_pb): - from gcloud_bigtable.column_family import _gc_rule_from_pb - return _gc_rule_from_pb(gc_rule_pb) - - def test_empty(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - - gc_rule_pb = data_pb2.GcRule() - self.assertEqual(self._callFUT(gc_rule_pb), None) - - def test_failure(self): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import duration_pb2 - - gc_rule_pb1 = data_pb2.GcRule(max_num_versions=1) - gc_rule_pb2 = data_pb2.GcRule( - max_age=duration_pb2.Duration(seconds=1), - ) - # Since a oneof field, google.protobuf doesn't allow both - # to be set, so we fake it. - gc_rule_pb3 = data_pb2.GcRule() - gc_rule_pb3._fields.update(gc_rule_pb1._fields) - gc_rule_pb3._fields.update(gc_rule_pb2._fields) - - with self.assertRaises(ValueError): - self._callFUT(gc_rule_pb3) - - def test_max_num_versions(self): - from gcloud_bigtable.column_family import GarbageCollectionRule - - orig_rule = GarbageCollectionRule(max_num_versions=1) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRule)) - self.assertEqual(result, orig_rule) - - def test_max_age(self): - import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - - orig_rule = GarbageCollectionRule( - max_age=datetime.timedelta(seconds=1)) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRule)) - self.assertEqual(result, orig_rule) - - def test_union(self): - import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import GarbageCollectionRuleUnion - - rule1 = GarbageCollectionRule(max_num_versions=1) - rule2 = GarbageCollectionRule( - max_age=datetime.timedelta(seconds=1)) - orig_rule = GarbageCollectionRuleUnion(rules=[rule1, rule2]) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRuleUnion)) - self.assertEqual(result, orig_rule) - - def test_intersection(self): - import datetime - from gcloud_bigtable.column_family import GarbageCollectionRule - from gcloud_bigtable.column_family import ( - GarbageCollectionRuleIntersection) - - rule1 = GarbageCollectionRule(max_num_versions=1) - rule2 = GarbageCollectionRule( - max_age=datetime.timedelta(seconds=1)) - orig_rule = GarbageCollectionRuleIntersection(rules=[rule1, rule2]) - gc_rule_pb = orig_rule.to_pb() - result = self._callFUT(gc_rule_pb) - self.assertTrue(isinstance(result, GarbageCollectionRuleIntersection)) - self.assertEqual(result, orig_rule) - - def test_unknown_field_name(self): - from google.protobuf.descriptor import FieldDescriptor - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - - gc_rule_pb = data_pb2.GcRule() - fake_descriptor_name = 'not-union' - descriptor_args = (fake_descriptor_name,) + (None,) * 12 - fake_descriptor = FieldDescriptor(*descriptor_args) - gc_rule_pb._fields[fake_descriptor] = None - self.assertEqual(self._callFUT(gc_rule_pb), None) - - -class _Client(object): - - cluster_stub = None - operations_stub = None - table_stub = None - - -class _Table(object): - - def __init__(self, name, client=None, timeout_seconds=None): - self.name = name - self.client = client - self.timeout_seconds = timeout_seconds diff --git a/gcloud_bigtable/test_row.py b/gcloud_bigtable/test_row.py deleted file mode 100644 index cc5556e..0000000 --- a/gcloud_bigtable/test_row.py +++ /dev/null @@ -1,1535 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT_ID = u'project-id' -ZONE = u'zone' -CLUSTER_ID = u'cluster-id' -TABLE_ID = u'table-id' -TABLE_NAME = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID + '/tables/' + TABLE_ID) -ROW_KEY = b'row_key' -ROW_KEY_NON_BYTES = u'row_key' -COLUMN = b'column' -COLUMN_NON_BYTES = u'column' -COLUMN_FAMILY_ID = u'column_family_id' - - -class TestRow(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import Row - return Row - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def _constructor_helper(self, row_key, row_key_expected=None, - filter_=None): - table = object() - row = self._makeOne(row_key, table, filter_=filter_) - row_key_val = row_key_expected or row_key - # Only necessary in Py2 - self.assertEqual(type(row._row_key), type(row_key_val)) - self.assertEqual(row._row_key, row_key_val) - self.assertTrue(row._table is table) - self.assertTrue(row._filter is filter_) - self.assertEqual(row._rule_pb_list, []) - if filter_ is None: - self.assertEqual(row._pb_mutations, []) - self.assertTrue(row._true_pb_mutations is None) - self.assertTrue(row._false_pb_mutations is None) - else: - self.assertTrue(row._pb_mutations is None) - self.assertEqual(row._true_pb_mutations, []) - self.assertEqual(row._false_pb_mutations, []) - - def test_constructor(self): - self._constructor_helper(ROW_KEY) - - def test_constructor_with_filter(self): - self._constructor_helper(ROW_KEY, filter_=object()) - - def test_constructor_with_unicode(self): - self._constructor_helper(ROW_KEY_NON_BYTES, row_key_expected=ROW_KEY) - - def test_constructor_with_non_bytes(self): - row_key = object() - with self.assertRaises(TypeError): - self._constructor_helper(row_key) - - def test_table_getter(self): - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertTrue(row.table is table) - - def test_row_key_getter(self): - row = self._makeOne(ROW_KEY, object()) - self.assertEqual(row.row_key, ROW_KEY) - - def test_filter_getter(self): - filter_ = object() - row = self._makeOne(ROW_KEY, object(), filter_=filter_) - self.assertTrue(row.filter is filter_) - - def test_client_getter(self): - client = object() - table = _Table(None, client=client) - row = self._makeOne(ROW_KEY, table) - self.assertTrue(row.client is client) - - def test_timeout_seconds_getter(self): - timeout_seconds = 889 - table = _Table(None, timeout_seconds=timeout_seconds) - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row.timeout_seconds, timeout_seconds) - - def _get_mutations_helper(self, filter_=None, state=None): - row = self._makeOne(ROW_KEY, None, filter_=filter_) - # Mock the mutations with unique objects so we can compare. - row._pb_mutations = no_bool = object() - row._true_pb_mutations = true_mutations = object() - row._false_pb_mutations = false_mutations = object() - - mutations = row._get_mutations(state) - return (no_bool, true_mutations, false_mutations), mutations - - def test__get_mutations_no_filter(self): - (no_bool, _, _), mutations = self._get_mutations_helper() - self.assertTrue(mutations is no_bool) - - def test__get_mutations_no_filter_bad_state(self): - state = object() # State should be null when no filter. - with self.assertRaises(ValueError): - self._get_mutations_helper(state=state) - - def test__get_mutations_with_filter_true_state(self): - filter_ = object() - state = True - (_, true_filter, _), mutations = self._get_mutations_helper( - filter_=filter_, state=state) - self.assertTrue(mutations is true_filter) - - def test__get_mutations_with_filter_false_state(self): - filter_ = object() - state = False - (_, _, false_filter), mutations = self._get_mutations_helper( - filter_=filter_, state=state) - self.assertTrue(mutations is false_filter) - - def test__get_mutations_with_filter_bad_state(self): - filter_ = object() - state = None - with self.assertRaises(ValueError): - self._get_mutations_helper(filter_=filter_, state=state) - - def test_append_cell_value(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._rule_pb_list, []) - - value = b'bytes-val' - row.append_cell_value(COLUMN_FAMILY_ID, COLUMN, value) - expected_pb = data_pb2.ReadModifyWriteRule( - family_name=COLUMN_FAMILY_ID, column_qualifier=COLUMN, - append_value=value) - self.assertEqual(row._rule_pb_list, [expected_pb]) - - def test_increment_cell_value(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._rule_pb_list, []) - - int_value = 281330 - row.increment_cell_value(COLUMN_FAMILY_ID, COLUMN, int_value) - expected_pb = data_pb2.ReadModifyWriteRule( - family_name=COLUMN_FAMILY_ID, column_qualifier=COLUMN, - increment_amount=int_value) - self.assertEqual(row._rule_pb_list, [expected_pb]) - - def _set_cell_helper(self, column=COLUMN, column_bytes=None, - value=b'foobar', timestamp=None, - timestamp_micros=-1): - import six - import struct - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._pb_mutations, []) - row.set_cell(COLUMN_FAMILY_ID, column, - value, timestamp=timestamp) - - if isinstance(value, six.integer_types): - value = struct.pack('>q', value) - expected_pb = data_pb2.Mutation( - set_cell=data_pb2.Mutation.SetCell( - family_name=COLUMN_FAMILY_ID, - column_qualifier=column_bytes or column, - timestamp_micros=timestamp_micros, - value=value, - ), - ) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_set_cell(self): - self._set_cell_helper(column=COLUMN) - - def test_set_cell_with_string_column(self): - self._set_cell_helper(column=COLUMN_NON_BYTES, column_bytes=COLUMN) - - def test_set_cell_with_integer_value(self): - value = 1337 - self._set_cell_helper(column=COLUMN, value=value) - - def test_set_cell_with_non_bytes_value(self): - table = object() - row = self._makeOne(ROW_KEY, table) - value = object() # Not bytes - with self.assertRaises(TypeError): - row.set_cell(COLUMN_FAMILY_ID, COLUMN, value) - - def test_set_cell_with_non_null_timestamp(self): - import datetime - from gcloud_bigtable. _helpers import EPOCH - - microseconds = 898294371 - millis_granularity = microseconds - (microseconds % 1000) - timestamp = EPOCH + datetime.timedelta(microseconds=microseconds) - self._set_cell_helper(timestamp=timestamp, - timestamp_micros=millis_granularity) - - def test_delete(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - row = self._makeOne(ROW_KEY, object()) - self.assertEqual(row._pb_mutations, []) - row.delete() - - expected_pb = data_pb2.Mutation( - delete_from_row=data_pb2.Mutation.DeleteFromRow(), - ) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_delete_cell(self): - klass = self._getTargetClass() - - class MockRow(klass): - - def __init__(self, *args, **kwargs): - super(MockRow, self).__init__(*args, **kwargs) - self._args = [] - self._kwargs = [] - - # Replace the called method with one that logs arguments. - def delete_cells(self, *args, **kwargs): - self._args.append(args) - self._kwargs.append(kwargs) - - table = object() - mock_row = MockRow(ROW_KEY, table) - # Make sure no values are set before calling the method. - self.assertEqual(mock_row._pb_mutations, []) - self.assertEqual(mock_row._args, []) - self.assertEqual(mock_row._kwargs, []) - - # Actually make the request against the mock class. - time_range = object() - mock_row.delete_cell(COLUMN_FAMILY_ID, COLUMN, time_range=time_range) - self.assertEqual(mock_row._pb_mutations, []) - self.assertEqual(mock_row._args, [(COLUMN_FAMILY_ID, [COLUMN])]) - self.assertEqual(mock_row._kwargs, [{ - 'state': None, - 'time_range': time_range, - }]) - - def test_delete_cells_non_iterable(self): - table = object() - row = self._makeOne(ROW_KEY, table) - columns = object() # Not iterable - with self.assertRaises(TypeError): - row.delete_cells(COLUMN_FAMILY_ID, columns) - - def test_delete_cells_all_columns(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - klass = self._getTargetClass() - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, klass.ALL_COLUMNS) - - expected_pb = data_pb2.Mutation( - delete_from_family=data_pb2.Mutation.DeleteFromFamily( - family_name=COLUMN_FAMILY_ID, - ), - ) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_delete_cells_no_columns(self): - table = object() - row = self._makeOne(ROW_KEY, table) - columns = [] - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, columns) - self.assertEqual(row._pb_mutations, []) - - def _delete_cells_helper(self, time_range=None): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - columns = [COLUMN] - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, columns, time_range=time_range) - - expected_pb = data_pb2.Mutation( - delete_from_column=data_pb2.Mutation.DeleteFromColumn( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - ), - ) - if time_range is not None: - expected_pb.delete_from_column.time_range.CopyFrom( - time_range.to_pb()) - self.assertEqual(row._pb_mutations, [expected_pb]) - - def test_delete_cells_no_time_range(self): - self._delete_cells_helper() - - def test_delete_cells_with_time_range(self): - import datetime - from gcloud_bigtable. _helpers import EPOCH - from gcloud_bigtable.row import TimestampRange - - microseconds = 30871000 # Makes sure already milliseconds granularity - start = EPOCH + datetime.timedelta(microseconds=microseconds) - time_range = TimestampRange(start=start) - self._delete_cells_helper(time_range=time_range) - - def test_delete_cells_with_bad_column(self): - # This makes sure a failure on one of the columns doesn't leave - # the row's mutations in a bad state. - table = object() - row = self._makeOne(ROW_KEY, table) - columns = [COLUMN, object()] - self.assertEqual(row._pb_mutations, []) - with self.assertRaises(TypeError): - row.delete_cells(COLUMN_FAMILY_ID, columns) - self.assertEqual(row._pb_mutations, []) - - def test_delete_cells_with_string_columns(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - table = object() - row = self._makeOne(ROW_KEY, table) - column1 = u'column1' - column1_bytes = b'column1' - column2 = u'column2' - column2_bytes = b'column2' - columns = [column1, column2] - self.assertEqual(row._pb_mutations, []) - row.delete_cells(COLUMN_FAMILY_ID, columns) - - expected_pb1 = data_pb2.Mutation( - delete_from_column=data_pb2.Mutation.DeleteFromColumn( - family_name=COLUMN_FAMILY_ID, - column_qualifier=column1_bytes, - ), - ) - expected_pb2 = data_pb2.Mutation( - delete_from_column=data_pb2.Mutation.DeleteFromColumn( - family_name=COLUMN_FAMILY_ID, - column_qualifier=column2_bytes, - ), - ) - self.assertEqual(row._pb_mutations, [expected_pb1, expected_pb2]) - - def test_commit(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table = _Table(TABLE_NAME, client=client) - row = self._makeOne(ROW_KEY, table) - - # Create request_pb - value = b'bytes-value' - mutation = data_pb2.Mutation( - set_cell=data_pb2.Mutation.SetCell( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - timestamp_micros=-1, # Default value. - value=value, - ), - ) - request_pb = messages_pb2.MutateRowRequest( - table_name=TABLE_NAME, - row_key=ROW_KEY, - mutations=[mutation], - ) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # commit() has no return value when no filter. - - # Perform the method and check the result. - timeout_seconds = 711 - row.set_cell(COLUMN_FAMILY_ID, COLUMN, value) - result = row.commit(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'MutateRow', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(row._pb_mutations, []) - self.assertEqual(row._true_pb_mutations, None) - self.assertEqual(row._false_pb_mutations, None) - - def test_commit_too_many_mutations(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import row as MUT - - table = object() - row = self._makeOne(ROW_KEY, table) - row._pb_mutations = [1, 2, 3] - num_mutations = len(row._pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): - with self.assertRaises(ValueError): - row.commit() - - def test_commit_no_mutations(self): - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table = _Table(None, client=client) - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._pb_mutations, []) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock() - - # Perform the method and check the result. - result = row.commit() - self.assertEqual(result, None) - # Make sure no request was sent. - self.assertEqual(stub.method_calls, []) - - def test_commit_with_filter(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable.row import RowFilter - - client = _Client() - table = _Table(TABLE_NAME, client=client) - row_filter = RowFilter(row_sample_filter=0.33) - row = self._makeOne(ROW_KEY, table, filter_=row_filter) - - # Create request_pb - value = b'bytes-value' - mutation = data_pb2.Mutation( - set_cell=data_pb2.Mutation.SetCell( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - timestamp_micros=-1, # Default value. - value=value, - ), - ) - request_pb = messages_pb2.CheckAndMutateRowRequest( - table_name=TABLE_NAME, - row_key=ROW_KEY, - predicate_filter=row_filter.to_pb(), - true_mutations=[mutation], - false_mutations=[], - ) - - # Create response_pb - predicate_matched = True - response_pb = messages_pb2.CheckAndMutateRowResponse( - predicate_matched=predicate_matched) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = predicate_matched - - # Perform the method and check the result. - timeout_seconds = 262 - row.set_cell(COLUMN_FAMILY_ID, COLUMN, value, state=True) - result = row.commit(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CheckAndMutateRow', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(row._pb_mutations, None) - self.assertEqual(row._true_pb_mutations, []) - self.assertEqual(row._false_pb_mutations, []) - - def test_commit_with_filter_too_many_mutations(self): - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import row as MUT - - table = object() - filter_ = object() - row = self._makeOne(ROW_KEY, table, filter_=filter_) - row._true_pb_mutations = [1, 2, 3] - num_mutations = len(row._true_pb_mutations) - with _Monkey(MUT, _MAX_MUTATIONS=num_mutations - 1): - with self.assertRaises(ValueError): - row.commit() - - def test_commit_with_filter_no_mutations(self): - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table = _Table(None, client=client) - filter_ = object() - row = self._makeOne(ROW_KEY, table, filter_=filter_) - self.assertEqual(row._true_pb_mutations, []) - self.assertEqual(row._false_pb_mutations, []) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock() - - # Perform the method and check the result. - result = row.commit() - self.assertEqual(result, None) - # Make sure no request was sent. - self.assertEqual(stub.method_calls, []) - - def test_commit_modifications(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable import row as MUT - - client = _Client() - table = _Table(TABLE_NAME, client=client) - row = self._makeOne(ROW_KEY, table) - - # Create request_pb - value = b'bytes-value' - # We will call row.append_cell_value(COLUMN_FAMILY_ID, COLUMN, value). - request_pb = messages_pb2.ReadModifyWriteRowRequest( - table_name=TABLE_NAME, - row_key=ROW_KEY, - rules=[ - data_pb2.ReadModifyWriteRule( - family_name=COLUMN_FAMILY_ID, - column_qualifier=COLUMN, - append_value=value, - ), - ], - ) - - # Create response_pb - response_pb = object() - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = object() - mock_parse_rmw_row_response = _MockCalled(expected_result) - - # Perform the method and check the result. - timeout_seconds = 87 - - with _Monkey(MUT, _parse_rmw_row_response=mock_parse_rmw_row_response): - row.append_cell_value(COLUMN_FAMILY_ID, COLUMN, value) - result = row.commit_modifications(timeout_seconds=timeout_seconds) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ReadModifyWriteRow', - (request_pb, timeout_seconds), - {}, - )]) - self.assertEqual(row._pb_mutations, []) - self.assertEqual(row._true_pb_mutations, None) - self.assertEqual(row._false_pb_mutations, None) - - mock_parse_rmw_row_response.check_called(self, [(response_pb,)]) - self.assertEqual(row._rule_pb_list, []) - - def test_commit_modifications_no_rules(self): - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - table = _Table(None, client=client) - row = self._makeOne(ROW_KEY, table) - self.assertEqual(row._rule_pb_list, []) - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock() - - # Perform the method and check the result. - result = row.commit_modifications() - self.assertEqual(result, {}) - # Make sure no request was sent. - self.assertEqual(stub.method_calls, []) - - -class Test__parse_rmw_row_response(unittest2.TestCase): - - def _callFUT(self, row_response): - from gcloud_bigtable.row import _parse_rmw_row_response - return _parse_rmw_row_response(row_response) - - def test_it(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._helpers import _microseconds_to_timestamp - - COL_FAM1 = u'col-fam-id' - COL_FAM2 = u'col-fam-id2' - COL_NAME1 = b'col-name1' - COL_NAME2 = b'col-name2' - COL_NAME3 = b'col-name3-but-other-fam' - CELL_VAL1 = b'cell-val' - CELL_VAL2 = b'cell-val-newer' - CELL_VAL3 = b'altcol-cell-val' - CELL_VAL4 = b'foo' - - microseconds = 1000871 - timestamp = _microseconds_to_timestamp(microseconds) - expected_output = { - COL_FAM1: { - COL_NAME1: [ - (CELL_VAL1, timestamp), - (CELL_VAL2, timestamp), - ], - COL_NAME2: [ - (CELL_VAL3, timestamp), - ], - }, - COL_FAM2: { - COL_NAME3: [ - (CELL_VAL4, timestamp), - ], - }, - } - sample_input = data_pb2.Row( - families=[ - data_pb2.Family( - name=COL_FAM1, - columns=[ - data_pb2.Column( - qualifier=COL_NAME1, - cells=[ - data_pb2.Cell( - value=CELL_VAL1, - timestamp_micros=microseconds, - ), - data_pb2.Cell( - value=CELL_VAL2, - timestamp_micros=microseconds, - ), - ], - ), - data_pb2.Column( - qualifier=COL_NAME2, - cells=[ - data_pb2.Cell( - value=CELL_VAL3, - timestamp_micros=microseconds, - ), - ], - ), - ], - ), - data_pb2.Family( - name=COL_FAM2, - columns=[ - data_pb2.Column( - qualifier=COL_NAME3, - cells=[ - data_pb2.Cell( - value=CELL_VAL4, - timestamp_micros=microseconds, - ), - ], - ), - ], - ), - ], - ) - self.assertEqual(expected_output, self._callFUT(sample_input)) - - -class TestRowFilter(unittest2.TestCase): - - _PROPERTIES = ( - 'sink', - 'pass_all_filter', - 'block_all_filter', - 'row_key_regex_filter', - 'row_sample_filter', - 'family_name_regex_filter', - 'column_qualifier_regex_filter', - 'column_range_filter', - 'timestamp_range_filter', - 'value_regex_filter', - 'value_range_filter', - 'cells_per_row_offset_filter', - 'cells_per_row_limit_filter', - 'cells_per_column_limit_filter', - 'strip_value_transformer', - 'apply_label_transformer', - ) - - def _getTargetClass(self): - from gcloud_bigtable.row import RowFilter - return RowFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - # Fails unless exactly one property is passed. - with self.assertRaises(TypeError): - self._makeOne() - - def test_constructor_too_many_values(self): - # Fails unless exactly one property is passed. - with self.assertRaises(TypeError): - self._makeOne(row_key_regex_filter=b'value', - cells_per_column_limit_filter=10) - - def _row_filter_values_check(self, row_filter, value_name, value): - for prop_name in self._PROPERTIES: - prop_value = getattr(row_filter, prop_name) - if prop_name == value_name: - self.assertEqual(prop_value, value) - else: - self.assertTrue(prop_value is None) - - def test_constructor_sink(self): - value = True - row_filter = self._makeOne(sink=value) - self._row_filter_values_check(row_filter, 'sink', value) - - def test_constructor_pass_all_filter(self): - value = True - row_filter = self._makeOne(pass_all_filter=value) - self._row_filter_values_check(row_filter, 'pass_all_filter', value) - - def test_constructor_block_all_filter(self): - value = True - row_filter = self._makeOne(block_all_filter=value) - self._row_filter_values_check(row_filter, 'block_all_filter', value) - - def test_constructor_row_key_regex_filter(self): - value = b'row-key-regex' - row_filter = self._makeOne(row_key_regex_filter=value) - self._row_filter_values_check( - row_filter, 'row_key_regex_filter', value) - - def test_constructor_row_sample_filter(self): - value = 0.25 - row_filter = self._makeOne(row_sample_filter=value) - self._row_filter_values_check(row_filter, 'row_sample_filter', value) - - def test_constructor_family_name_regex_filter(self): - value = u'family-regex' - row_filter = self._makeOne(family_name_regex_filter=value) - self._row_filter_values_check( - row_filter, 'family_name_regex_filter', value) - - def test_constructor_column_qualifier_regex_filter(self): - value = b'column-regex' - row_filter = self._makeOne(column_qualifier_regex_filter=value) - self._row_filter_values_check( - row_filter, 'column_qualifier_regex_filter', value) - - def test_constructor_column_range_filter(self): - from gcloud_bigtable.row import ColumnRange - value = ColumnRange(COLUMN_FAMILY_ID) - row_filter = self._makeOne(column_range_filter=value) - self._row_filter_values_check(row_filter, 'column_range_filter', - value) - - def test_constructor_timestamp_range_filter(self): - from gcloud_bigtable.row import TimestampRange - value = TimestampRange() - row_filter = self._makeOne(timestamp_range_filter=value) - self._row_filter_values_check(row_filter, 'timestamp_range_filter', - value) - - def test_constructor_value_regex_filter(self): - value = b'value-regex' - row_filter = self._makeOne(value_regex_filter=value) - self._row_filter_values_check(row_filter, 'value_regex_filter', value) - - def test_constructor_value_range_filter(self): - from gcloud_bigtable.row import CellValueRange - value = CellValueRange() - row_filter = self._makeOne(value_range_filter=value) - self._row_filter_values_check(row_filter, 'value_range_filter', - value) - - def test_constructor_cells_per_row_offset_filter(self): - value = 76 - row_filter = self._makeOne(cells_per_row_offset_filter=value) - self._row_filter_values_check( - row_filter, 'cells_per_row_offset_filter', value) - - def test_constructor_cells_per_row_limit_filter(self): - value = 189 - row_filter = self._makeOne(cells_per_row_limit_filter=value) - self._row_filter_values_check( - row_filter, 'cells_per_row_limit_filter', value) - - def test_constructor_cells_per_column_limit_filter(self): - value = 10 - row_filter = self._makeOne(cells_per_column_limit_filter=value) - self._row_filter_values_check( - row_filter, 'cells_per_column_limit_filter', value) - - def test_constructor_strip_value_transformer(self): - value = True - row_filter = self._makeOne(strip_value_transformer=value) - self._row_filter_values_check(row_filter, 'strip_value_transformer', - value) - - def test_constructor_apply_label_transformer(self): - value = u'label' - row_filter = self._makeOne(apply_label_transformer=value) - self._row_filter_values_check(row_filter, 'apply_label_transformer', - value) - - def test___eq__(self): - # Fool the constructor by passing exactly 1 value. - row_filter1 = self._makeOne(strip_value_transformer=True) - row_filter2 = self._makeOne(strip_value_transformer=True) - # Set every value so we can compare them all. - for prop_name in self._PROPERTIES: - fake_val = object() - setattr(row_filter1, prop_name, fake_val) - setattr(row_filter2, prop_name, fake_val) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - # Fool the constructor by passing exactly 1 value. - row_filter1 = self._makeOne(strip_value_transformer=True) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - # Fool the constructor by passing exactly 1 value. - row_filter1 = self._makeOne(strip_value_transformer=True) - row_filter2 = self._makeOne(strip_value_transformer=True) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - def test_to_pb_empty(self): - # Fool the constructor by passing exactly 1 value. - row_filter = self._makeOne(strip_value_transformer=True) - # Make it artificially empty after the fact. - row_filter.strip_value_transformer = None - - with self.assertRaises(TypeError): - row_filter.to_pb() - - def _to_pb_test_helper(self, **kwargs): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - row_filter = self._makeOne(**kwargs) - - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(**kwargs) - self.assertEqual(pb_val, expected_pb) - - def test_to_pb_with_sink(self): - value = True - self._to_pb_test_helper(sink=value) - - def test_to_pb_with_pass_all_filter(self): - value = True - self._to_pb_test_helper(pass_all_filter=value) - - def test_to_pb_with_block_all_filter(self): - value = True - self._to_pb_test_helper(block_all_filter=value) - - def test_to_pb_with_row_key_regex_filter(self): - value = b'row-key-regex' - self._to_pb_test_helper(row_key_regex_filter=value) - - def test_to_pb_with_row_sample_filter(self): - value = 0.25 - self._to_pb_test_helper(row_sample_filter=value) - - def test_to_pb_with_family_name_regex_filter(self): - value = u'family-regex' - self._to_pb_test_helper(family_name_regex_filter=value) - - def test_to_pb_with_column_qualifier_regex_filter(self): - value = b'column-regex' - self._to_pb_test_helper(column_qualifier_regex_filter=value) - - def test_to_pb_with_column_range_filter(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import ColumnRange - - value = ColumnRange(COLUMN_FAMILY_ID) - row_filter = self._makeOne(column_range_filter=value) - - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(column_range_filter=value.to_pb()) - self.assertEqual(pb_val, expected_pb) - - def test_to_pb_with_timestamp_range_filter(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import TimestampRange - - value = TimestampRange() - row_filter = self._makeOne(timestamp_range_filter=value) - - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(timestamp_range_filter=value.to_pb()) - self.assertEqual(pb_val, expected_pb) - - def test_to_pb_with_value_regex_filter(self): - value = b'value-regex' - self._to_pb_test_helper(value_regex_filter=value) - - def test_to_pb_with_value_range_filter(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import CellValueRange - - value = CellValueRange() - row_filter = self._makeOne(value_range_filter=value) - - pb_val = row_filter.to_pb() - expected_pb = data_pb2.RowFilter(value_range_filter=value.to_pb()) - self.assertEqual(pb_val, expected_pb) - - def test_to_pb_with_cells_per_row_offset_filter(self): - value = 76 - self._to_pb_test_helper(cells_per_row_offset_filter=value) - - def test_to_pb_with_cells_per_row_limit_filter(self): - value = 189 - self._to_pb_test_helper(cells_per_row_limit_filter=value) - - def test_to_pb_with_cells_per_column_limit_filter(self): - value = 10 - self._to_pb_test_helper(cells_per_column_limit_filter=value) - - def test_to_pb_with_strip_value_transformer(self): - value = True - self._to_pb_test_helper(strip_value_transformer=value) - - def test_to_pb_with_apply_label_transformer(self): - value = u'label' - self._to_pb_test_helper(apply_label_transformer=value) - - -class TestTimestampRange(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import TimestampRange - return TimestampRange - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - start = object() - end = object() - time_range = self._makeOne(start=start, end=end) - self.assertTrue(time_range.start is start) - self.assertTrue(time_range.end is end) - - def test___eq__(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - self.assertEqual(time_range1, time_range2) - - def test___eq__type_differ(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = object() - self.assertNotEqual(time_range1, time_range2) - - def test___ne__same_value(self): - start = object() - end = object() - time_range1 = self._makeOne(start=start, end=end) - time_range2 = self._makeOne(start=start, end=end) - comparison_val = (time_range1 != time_range2) - self.assertFalse(comparison_val) - - def _to_pb_helper(self, start_micros=None, end_micros=None): - import datetime - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable. _helpers import EPOCH - - pb_kwargs = {} - - start = None - if start_micros is not None: - start = EPOCH + datetime.timedelta(microseconds=start_micros) - pb_kwargs['start_timestamp_micros'] = start_micros - end = None - if end_micros is not None: - end = EPOCH + datetime.timedelta(microseconds=end_micros) - pb_kwargs['end_timestamp_micros'] = end_micros - time_range = self._makeOne(start=start, end=end) - - expected_pb = data_pb2.TimestampRange(**pb_kwargs) - self.assertEqual(time_range.to_pb(), expected_pb) - - def test_to_pb(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - end_micros = 12939371000 - self._to_pb_helper(start_micros=start_micros, - end_micros=end_micros) - - def test_to_pb_just_start(self): - # Makes sure already milliseconds granularity - start_micros = 30871000 - self._to_pb_helper(start_micros=start_micros) - - def test_to_pb_just_end(self): - # Makes sure already milliseconds granularity - end_micros = 12939371000 - self._to_pb_helper(end_micros=end_micros) - - -class TestColumnRange(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ColumnRange - return ColumnRange - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - column_family_id = object() - col_range = self._makeOne(column_family_id) - self.assertTrue(col_range.column_family_id is column_family_id) - self.assertEqual(col_range.start_column, None) - self.assertEqual(col_range.end_column, None) - self.assertTrue(col_range.inclusive_start) - self.assertTrue(col_range.inclusive_end) - - def test_constructor_explicit(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - col_range = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(col_range.column_family_id is column_family_id) - self.assertTrue(col_range.start_column is start_column) - self.assertTrue(col_range.end_column is end_column) - self.assertTrue(col_range.inclusive_start is inclusive_start) - self.assertTrue(col_range.inclusive_end is inclusive_end) - - def test___eq__(self): - column_family_id = object() - start_column = object() - end_column = object() - inclusive_start = object() - inclusive_end = object() - col_range1 = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - col_range2 = self._makeOne(column_family_id, start_column=start_column, - end_column=end_column, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(col_range1, col_range2) - - def test___eq__type_differ(self): - column_family_id = object() - col_range1 = self._makeOne(column_family_id) - col_range2 = object() - self.assertNotEqual(col_range1, col_range2) - - def test___ne__same_value(self): - column_family_id = object() - col_range1 = self._makeOne(column_family_id) - col_range2 = self._makeOne(column_family_id) - comparison_val = (col_range1 != col_range2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - col_range = self._makeOne(COLUMN_FAMILY_ID) - expected_pb = data_pb2.ColumnRange(family_name=COLUMN_FAMILY_ID) - self.assertEqual(col_range.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - col_range = self._makeOne(COLUMN_FAMILY_ID, start_column=COLUMN) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - start_qualifier_inclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - col_range = self._makeOne(COLUMN_FAMILY_ID, start_column=COLUMN, - inclusive_start=False) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - start_qualifier_exclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - col_range = self._makeOne(COLUMN_FAMILY_ID, end_column=COLUMN) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - end_qualifier_inclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - col_range = self._makeOne(COLUMN_FAMILY_ID, end_column=COLUMN, - inclusive_end=False) - expected_pb = data_pb2.ColumnRange( - family_name=COLUMN_FAMILY_ID, - end_qualifier_exclusive=COLUMN, - ) - self.assertEqual(col_range.to_pb(), expected_pb) - - -class TestCellValueRange(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import CellValueRange - return CellValueRange - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor_defaults(self): - val_range = self._makeOne() - self.assertEqual(val_range.start_value, None) - self.assertEqual(val_range.end_value, None) - self.assertTrue(val_range.inclusive_start) - self.assertTrue(val_range.inclusive_end) - - def test_constructor_explicit(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - val_range = self._makeOne(start_value=start_value, end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertTrue(val_range.start_value is start_value) - self.assertTrue(val_range.end_value is end_value) - self.assertTrue(val_range.inclusive_start is inclusive_start) - self.assertTrue(val_range.inclusive_end is inclusive_end) - - def test___eq__(self): - start_value = object() - end_value = object() - inclusive_start = object() - inclusive_end = object() - val_range1 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - val_range2 = self._makeOne(start_value=start_value, - end_value=end_value, - inclusive_start=inclusive_start, - inclusive_end=inclusive_end) - self.assertEqual(val_range1, val_range2) - - def test___eq__type_differ(self): - val_range1 = self._makeOne() - val_range2 = object() - self.assertNotEqual(val_range1, val_range2) - - def test___ne__same_value(self): - val_range1 = self._makeOne() - val_range2 = self._makeOne() - comparison_val = (val_range1 != val_range2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - val_range = self._makeOne() - expected_pb = data_pb2.ValueRange() - self.assertEqual(val_range.to_pb(), expected_pb) - - def test_to_pb_inclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - val_range = self._makeOne(start_value=value) - expected_pb = data_pb2.ValueRange(start_value_inclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) - - def test_to_pb_exclusive_start(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - val_range = self._makeOne(start_value=value, inclusive_start=False) - expected_pb = data_pb2.ValueRange(start_value_exclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) - - def test_to_pb_inclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - val_range = self._makeOne(end_value=value) - expected_pb = data_pb2.ValueRange(end_value_inclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) - - def test_to_pb_exclusive_end(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - - value = b'some-value' - val_range = self._makeOne(end_value=value, inclusive_end=False) - expected_pb = data_pb2.ValueRange(end_value_exclusive=value) - self.assertEqual(val_range.to_pb(), expected_pb) - - -class TestRowFilterChain(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import RowFilterChain - return RowFilterChain - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - filters = object() - filter_chain = self._makeOne(filters=filters) - self.assertTrue(filter_chain.filters is filters) - - def test___eq__(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter - - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowFilter(row_sample_filter=0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter - - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter2 = RowFilter(row_sample_filter=0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = RowFilter(cells_per_row_limit_filter=11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - chain=data_pb2.RowFilter.Chain( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestRowFilterUnion(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import RowFilterUnion - return RowFilterUnion - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - filters = object() - filter_union = self._makeOne(filters=filters) - self.assertTrue(filter_union.filters is filters) - - def test___eq__(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - self.assertEqual(row_filter1, row_filter2) - - def test___eq__type_differ(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = object() - self.assertNotEqual(row_filter1, row_filter2) - - def test___ne__same_value(self): - filters = object() - row_filter1 = self._makeOne(filters=filters) - row_filter2 = self._makeOne(filters=filters) - comparison_val = (row_filter1 != row_filter2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter - - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowFilter(row_sample_filter=0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter1_pb, row_filter2_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_nested(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter - - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter2 = RowFilter(row_sample_filter=0.25) - - row_filter3 = self._makeOne(filters=[row_filter1, row_filter2]) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = RowFilter(cells_per_row_limit_filter=11) - row_filter4_pb = row_filter4.to_pb() - - row_filter5 = self._makeOne(filters=[row_filter3, row_filter4]) - filter_pb = row_filter5.to_pb() - - expected_pb = data_pb2.RowFilter( - interleave=data_pb2.RowFilter.Interleave( - filters=[row_filter3_pb, row_filter4_pb], - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class TestConditionalRowFilter(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row import ConditionalRowFilter - return ConditionalRowFilter - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertTrue(cond_filter.base_filter is base_filter) - self.assertTrue(cond_filter.true_filter is true_filter) - self.assertTrue(cond_filter.false_filter is false_filter) - - def test___eq__(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - self.assertEqual(cond_filter1, cond_filter2) - - def test___eq__type_differ(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = object() - self.assertNotEqual(cond_filter1, cond_filter2) - - def test___ne__same_value(self): - base_filter = object() - true_filter = object() - false_filter = object() - cond_filter1 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - cond_filter2 = self._makeOne(base_filter, - true_filter=true_filter, - false_filter=false_filter) - comparison_val = (cond_filter1 != cond_filter2) - self.assertFalse(comparison_val) - - def test_to_pb(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter - - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowFilter(row_sample_filter=0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = RowFilter(cells_per_row_limit_filter=11) - row_filter3_pb = row_filter3.to_pb() - - row_filter4 = self._makeOne(row_filter1, true_filter=row_filter2, - false_filter=row_filter3) - filter_pb = row_filter4.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - false_filter=row_filter3_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_true_only(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter - - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowFilter(row_sample_filter=0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, true_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - true_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - def test_to_pb_false_only(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable.row import RowFilter - - row_filter1 = RowFilter(strip_value_transformer=True) - row_filter1_pb = row_filter1.to_pb() - - row_filter2 = RowFilter(row_sample_filter=0.25) - row_filter2_pb = row_filter2.to_pb() - - row_filter3 = self._makeOne(row_filter1, false_filter=row_filter2) - filter_pb = row_filter3.to_pb() - - expected_pb = data_pb2.RowFilter( - condition=data_pb2.RowFilter.Condition( - predicate_filter=row_filter1_pb, - false_filter=row_filter2_pb, - ), - ) - self.assertEqual(filter_pb, expected_pb) - - -class _Client(object): - - data_stub = None - - -class _Table(object): - - def __init__(self, name, client=None, timeout_seconds=None): - self.name = name - self.client = client - self.timeout_seconds = timeout_seconds diff --git a/gcloud_bigtable/test_row_data.py b/gcloud_bigtable/test_row_data.py deleted file mode 100644 index 5a29433..0000000 --- a/gcloud_bigtable/test_row_data.py +++ /dev/null @@ -1,527 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -class TestCell(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row_data import Cell - return Cell - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def _from_pb_test_helper(self, labels=None): - import datetime - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._helpers import EPOCH - - timestamp_micros = 18738724000 # Make sure millis granularity - timestamp = EPOCH + datetime.timedelta(microseconds=timestamp_micros) - value = b'value-bytes' - - if labels is None: - cell_pb = data_pb2.Cell(value=value, - timestamp_micros=timestamp_micros) - cell_expected = self._makeOne(value, timestamp) - else: - cell_pb = data_pb2.Cell(value=value, - timestamp_micros=timestamp_micros, - labels=labels) - cell_expected = self._makeOne(value, timestamp, labels=labels) - - klass = self._getTargetClass() - result = klass.from_pb(cell_pb) - self.assertEqual(result, cell_expected) - - def test_from_pb(self): - self._from_pb_test_helper() - - def test_from_pb_with_labels(self): - labels = [u'label1', u'label2'] - self._from_pb_test_helper(labels) - - def test_constructor(self): - value = object() - timestamp = object() - cell = self._makeOne(value, timestamp) - self.assertEqual(cell.value, value) - self.assertEqual(cell.timestamp, timestamp) - - def test___eq__(self): - value = object() - timestamp = object() - cell1 = self._makeOne(value, timestamp) - cell2 = self._makeOne(value, timestamp) - self.assertEqual(cell1, cell2) - - def test___eq__type_differ(self): - cell1 = self._makeOne(None, None) - cell2 = object() - self.assertNotEqual(cell1, cell2) - - def test___ne__same_value(self): - value = object() - timestamp = object() - cell1 = self._makeOne(value, timestamp) - cell2 = self._makeOne(value, timestamp) - comparison_val = (cell1 != cell2) - self.assertFalse(comparison_val) - - def test___ne__(self): - value1 = 'value1' - value2 = 'value2' - timestamp = object() - cell1 = self._makeOne(value1, timestamp) - cell2 = self._makeOne(value2, timestamp) - self.assertNotEqual(cell1, cell2) - - -class TestPartialRowData(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row_data import PartialRowData - return PartialRowData - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - row_key = object() - partial_row_data = self._makeOne(row_key) - self.assertTrue(partial_row_data._row_key is row_key) - self.assertEqual(partial_row_data._cells, {}) - self.assertFalse(partial_row_data._committed) - self.assertFalse(partial_row_data._chunks_encountered) - - def test___eq__(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data2 = self._makeOne(row_key) - self.assertEqual(partial_row_data1, partial_row_data2) - - def test___eq__type_differ(self): - partial_row_data1 = self._makeOne(None) - partial_row_data2 = object() - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test___ne__same_value(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data2 = self._makeOne(row_key) - comparison_val = (partial_row_data1 != partial_row_data2) - self.assertFalse(comparison_val) - - def test___ne__(self): - row_key1 = object() - partial_row_data1 = self._makeOne(row_key1) - row_key2 = object() - partial_row_data2 = self._makeOne(row_key2) - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test___ne__committed(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data1._committed = object() - partial_row_data2 = self._makeOne(row_key) - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test___ne__cells(self): - row_key = object() - partial_row_data1 = self._makeOne(row_key) - partial_row_data1._cells = object() - partial_row_data2 = self._makeOne(row_key) - self.assertNotEqual(partial_row_data1, partial_row_data2) - - def test_to_dict(self): - cell1 = object() - cell2 = object() - cell3 = object() - - family_name1 = u'name1' - family_name2 = u'name2' - qual1 = b'col1' - qual2 = b'col2' - qual3 = b'col3' - - partial_row_data = self._makeOne(None) - partial_row_data._cells = { - family_name1: { - qual1: cell1, - qual2: cell2, - }, - family_name2: { - qual3: cell3, - }, - } - - result = partial_row_data.to_dict() - col1 = family_name1.encode('ascii') + b':' + qual1 - col2 = family_name1.encode('ascii') + b':' + qual2 - col3 = family_name2.encode('ascii') + b':' + qual3 - expected_result = { - col1: cell1, - col2: cell2, - col3: cell3, - } - self.assertEqual(result, expected_result) - - def test_cells_property(self): - partial_row_data = self._makeOne(None) - cells = {1: 2} - partial_row_data._cells = cells - # Make sure we get a copy, not the original. - self.assertFalse(partial_row_data.cells is cells) - self.assertEqual(partial_row_data.cells, cells) - - def test_row_key_getter(self): - row_key = object() - partial_row_data = self._makeOne(row_key) - self.assertTrue(partial_row_data.row_key is row_key) - - def test_committed_getter(self): - partial_row_data = self._makeOne(None) - partial_row_data._committed = value = object() - self.assertTrue(partial_row_data.committed is value) - - def test_clear(self): - partial_row_data = self._makeOne(None) - cells = {1: 2} - partial_row_data._cells = cells - self.assertEqual(partial_row_data.cells, cells) - partial_row_data._committed = True - partial_row_data._chunks_encountered = True - partial_row_data.clear() - self.assertFalse(partial_row_data.committed) - self.assertFalse(partial_row_data._chunks_encountered) - self.assertEqual(partial_row_data.cells, {}) - - def test__handle_commit_row(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - - index = last_chunk_index = 1 - self.assertFalse(partial_row_data.committed) - partial_row_data._handle_commit_row(chunk, index, last_chunk_index) - self.assertTrue(partial_row_data.committed) - - def test__handle_commit_row_false(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=False) - - with self.assertRaises(ValueError): - partial_row_data._handle_commit_row(chunk, None, None) - - def test__handle_commit_row_not_last_chunk(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - - with self.assertRaises(ValueError): - index = 0 - last_chunk_index = 1 - self.assertNotEqual(index, last_chunk_index) - partial_row_data._handle_commit_row(chunk, index, last_chunk_index) - - def test__handle_reset_row(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(reset_row=True) - - # Modify the PartialRowData object so we can check it's been cleared. - partial_row_data._cells = {1: 2} - partial_row_data._committed = True - partial_row_data._handle_reset_row(chunk) - self.assertEqual(partial_row_data.cells, {}) - self.assertFalse(partial_row_data.committed) - - def test__handle_reset_row_failure(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - partial_row_data = self._makeOne(None) - chunk = messages_pb2.ReadRowsResponse.Chunk(reset_row=False) - - with self.assertRaises(ValueError): - partial_row_data._handle_reset_row(chunk) - - def test__handle_row_contents(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row_data import Cell - - partial_row_data = self._makeOne(None) - cell1_pb = data_pb2.Cell(timestamp_micros=1, value=b'val1') - cell2_pb = data_pb2.Cell(timestamp_micros=200, value=b'val2') - cell3_pb = data_pb2.Cell(timestamp_micros=300000, value=b'val3') - col1 = b'col1' - col2 = b'col2' - columns = [ - data_pb2.Column(qualifier=col1, cells=[cell1_pb, cell2_pb]), - data_pb2.Column(qualifier=col2, cells=[cell3_pb]), - ] - family_name = u'name' - row_contents = data_pb2.Family(name=family_name, columns=columns) - chunk = messages_pb2.ReadRowsResponse.Chunk(row_contents=row_contents) - - self.assertEqual(partial_row_data.cells, {}) - partial_row_data._handle_row_contents(chunk) - expected_cells = { - family_name: { - col1: [Cell.from_pb(cell1_pb), Cell.from_pb(cell2_pb)], - col2: [Cell.from_pb(cell3_pb)], - } - } - self.assertEqual(partial_row_data.cells, expected_cells) - - def test_update_from_read_rows(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - row_key = b'row-key' - partial_row_data = self._makeOne(row_key) - - # Set-up chunk1, some data that will be reset by chunk2. - ignored_family_name = u'ignore-name' - row_contents = data_pb2.Family(name=ignored_family_name) - chunk1 = messages_pb2.ReadRowsResponse.Chunk(row_contents=row_contents) - - # Set-up chunk2, a reset row. - chunk2 = messages_pb2.ReadRowsResponse.Chunk(reset_row=True) - - # Set-up chunk3, a column family with no columns. - family_name = u'name' - row_contents = data_pb2.Family(name=family_name) - chunk3 = messages_pb2.ReadRowsResponse.Chunk(row_contents=row_contents) - - # Set-up chunk4, a commit row. - chunk4 = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - - # Prepare request and make sure PartialRowData is empty before. - read_rows_response_pb = messages_pb2.ReadRowsResponse( - row_key=row_key, chunks=[chunk1, chunk2, chunk3, chunk4]) - self.assertEqual(partial_row_data.cells, {}) - self.assertFalse(partial_row_data.committed) - self.assertFalse(partial_row_data._chunks_encountered) - - # Parse the response and make sure the cells took place. - partial_row_data.update_from_read_rows(read_rows_response_pb) - self.assertEqual(partial_row_data.cells, {family_name: {}}) - self.assertFalse(ignored_family_name in partial_row_data.cells) - self.assertTrue(partial_row_data.committed) - self.assertTrue(partial_row_data._chunks_encountered) - - def test_update_from_read_rows_while_committed(self): - partial_row_data = self._makeOne(None) - partial_row_data._committed = True - self.assertFalse(partial_row_data._chunks_encountered) - - with self.assertRaises(ValueError): - partial_row_data.update_from_read_rows(None) - - self.assertFalse(partial_row_data._chunks_encountered) - - def test_update_from_read_rows_row_key_disagree(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - row_key1 = b'row-key1' - row_key2 = b'row-key2' - partial_row_data = self._makeOne(row_key1) - self.assertFalse(partial_row_data._chunks_encountered) - - self.assertNotEqual(row_key1, row_key2) - read_rows_response_pb = messages_pb2.ReadRowsResponse(row_key=row_key2) - with self.assertRaises(ValueError): - partial_row_data.update_from_read_rows(read_rows_response_pb) - - self.assertFalse(partial_row_data._chunks_encountered) - - def test_update_from_read_rows_empty_chunk(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - row_key = b'row-key' - partial_row_data = self._makeOne(row_key) - self.assertFalse(partial_row_data._chunks_encountered) - - chunk = messages_pb2.ReadRowsResponse.Chunk() - read_rows_response_pb = messages_pb2.ReadRowsResponse( - row_key=row_key, chunks=[chunk]) - - # This makes it an "empty" chunk. - self.assertEqual(chunk.WhichOneof('chunk'), None) - with self.assertRaises(ValueError): - partial_row_data.update_from_read_rows(read_rows_response_pb) - - self.assertFalse(partial_row_data._chunks_encountered) - - -class TestPartialRowsData(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.row_data import PartialRowsData - return PartialRowsData - - def _getDoNothingClass(self): - klass = self._getTargetClass() - - class FakePartialRowsData(klass): - - def __init__(self, *args, **kwargs): - super(FakePartialRowsData, self).__init__(*args, **kwargs) - self._consumed = [] - - def consume_next(self): - value = self._response_iterator.next() - self._consumed.append(value) - return value - - return FakePartialRowsData - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - response_iterator = object() - partial_rows_data = self._makeOne(response_iterator) - self.assertTrue(partial_rows_data._response_iterator - is response_iterator) - self.assertEqual(partial_rows_data._rows, {}) - - def test_rows_getter(self): - partial_rows_data = self._makeOne(None) - partial_rows_data._rows = value = object() - self.assertTrue(partial_rows_data.rows is value) - - def test___eq__(self): - response_iterator = object() - partial_rows_data1 = self._makeOne(response_iterator) - partial_rows_data2 = self._makeOne(response_iterator) - self.assertEqual(partial_rows_data1, partial_rows_data2) - - def test___eq__type_differ(self): - partial_rows_data1 = self._makeOne(None) - partial_rows_data2 = object() - self.assertNotEqual(partial_rows_data1, partial_rows_data2) - - def test___ne__same_value(self): - response_iterator = object() - partial_rows_data1 = self._makeOne(response_iterator) - partial_rows_data2 = self._makeOne(response_iterator) - comparison_val = (partial_rows_data1 != partial_rows_data2) - self.assertFalse(comparison_val) - - def test___ne__(self): - response_iterator1 = object() - partial_rows_data1 = self._makeOne(response_iterator1) - response_iterator2 = object() - partial_rows_data2 = self._makeOne(response_iterator2) - self.assertNotEqual(partial_rows_data1, partial_rows_data2) - - def test_cancel(self): - response_iterator = _MockCancellableIterator() - partial_rows_data = self._makeOne(response_iterator) - self.assertEqual(response_iterator.cancel_calls, 0) - partial_rows_data.cancel() - self.assertEqual(response_iterator.cancel_calls, 1) - - def test_consume_next(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row_data import PartialRowData - - row_key = b'row-key' - value_pb = messages_pb2.ReadRowsResponse(row_key=row_key) - response_iterator = _MockCancellableIterator(value_pb) - partial_rows_data = self._makeOne(response_iterator) - self.assertEqual(partial_rows_data.rows, {}) - partial_rows_data.consume_next() - expected_rows = {row_key: PartialRowData(row_key)} - self.assertEqual(partial_rows_data.rows, expected_rows) - - def test_consume_next_row_exists(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row_data import PartialRowData - - row_key = b'row-key' - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - value_pb = messages_pb2.ReadRowsResponse(row_key=row_key, - chunks=[chunk]) - response_iterator = _MockCancellableIterator(value_pb) - partial_rows_data = self._makeOne(response_iterator) - existing_values = PartialRowData(row_key) - partial_rows_data._rows[row_key] = existing_values - self.assertFalse(existing_values.committed) - partial_rows_data.consume_next() - self.assertTrue(existing_values.committed) - self.assertEqual(existing_values.cells, {}) - - def test_consume_next_empty_iter(self): - response_iterator = _MockCancellableIterator() - partial_rows_data = self._makeOne(response_iterator) - with self.assertRaises(StopIteration): - partial_rows_data.consume_next() - - def test_consume_all(self): - klass = self._getDoNothingClass() - - value1, value2, value3 = object(), object(), object() - response_iterator = _MockCancellableIterator(value1, value2, value3) - partial_rows_data = klass(response_iterator) - self.assertEqual(partial_rows_data._consumed, []) - partial_rows_data.consume_all() - self.assertEqual(partial_rows_data._consumed, [value1, value2, value3]) - - def test_consume_all_with_max_loops(self): - klass = self._getDoNothingClass() - - value1, value2, value3 = object(), object(), object() - response_iterator = _MockCancellableIterator(value1, value2, value3) - partial_rows_data = klass(response_iterator) - self.assertEqual(partial_rows_data._consumed, []) - partial_rows_data.consume_all(max_loops=1) - self.assertEqual(partial_rows_data._consumed, [value1]) - # Make sure the iterator still has the remaining values. - self.assertEqual(list(response_iterator.iter_values), [value2, value3]) - - -class _MockCancellableIterator(object): - - cancel_calls = 0 - - def __init__(self, *values): - self.iter_values = iter(values) - - def cancel(self): - self.cancel_calls += 1 - - def next(self): - return next(self.iter_values) diff --git a/gcloud_bigtable/test_table.py b/gcloud_bigtable/test_table.py deleted file mode 100644 index fa9b551..0000000 --- a/gcloud_bigtable/test_table.py +++ /dev/null @@ -1,592 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import unittest2 - - -PROJECT_ID = 'project-id' -ZONE = 'zone' -CLUSTER_ID = 'cluster-id' -TABLE_ID = 'table-id' - - -class TestTable(unittest2.TestCase): - - def _getTargetClass(self): - from gcloud_bigtable.table import Table - return Table - - def _makeOne(self, *args, **kwargs): - return self._getTargetClass()(*args, **kwargs) - - def test_constructor(self): - cluster = object() - table = self._makeOne(TABLE_ID, cluster) - self.assertEqual(table.table_id, TABLE_ID) - self.assertTrue(table._cluster is cluster) - - def test_cluster_getter(self): - cluster = object() - table = self._makeOne(TABLE_ID, cluster) - self.assertTrue(table.cluster is cluster) - - def test_client_getter(self): - client = object() - cluster = _Cluster(None, client=client) - table = self._makeOne(TABLE_ID, cluster) - self.assertTrue(table.client is client) - - def test_timeout_seconds_getter(self): - timeout_seconds = 1001 - cluster = _Cluster(None, timeout_seconds=timeout_seconds) - table = self._makeOne(TABLE_ID, cluster) - self.assertEqual(table.timeout_seconds, timeout_seconds) - - def test_name_property(self): - cluster_name = 'cluster_name' - cluster = _Cluster(cluster_name) - table = self._makeOne(TABLE_ID, cluster) - expected_name = cluster_name + '/tables/' + TABLE_ID - self.assertEqual(table.name, expected_name) - - def test_column_family_factory(self): - from gcloud_bigtable.column_family import ColumnFamily - - table = self._makeOne(TABLE_ID, None) - gc_rule = object() - column_family_id = 'column_family_id' - column_family = table.column_family(column_family_id, gc_rule=gc_rule) - self.assertTrue(isinstance(column_family, ColumnFamily)) - self.assertEqual(column_family.column_family_id, column_family_id) - self.assertTrue(column_family.gc_rule is gc_rule) - self.assertEqual(column_family._table, table) - - def test_row_factory(self): - from gcloud_bigtable.row import Row - - table = self._makeOne(TABLE_ID, None) - row_key = b'row_key' - filter_ = object() - row = table.row(row_key, filter_=filter_) - self.assertTrue(isinstance(row, Row)) - self.assertEqual(row.row_key, row_key) - self.assertEqual(row._table, table) - self.assertEqual(row._filter, filter_) - - def test___eq__(self): - table_id = 'table_id' - cluster = object() - table1 = self._makeOne(table_id, cluster) - table2 = self._makeOne(table_id, cluster) - self.assertEqual(table1, table2) - - def test___eq__type_differ(self): - table1 = self._makeOne('table_id', None) - table2 = object() - self.assertNotEqual(table1, table2) - - def test___ne__same_value(self): - table_id = 'table_id' - cluster = object() - table1 = self._makeOne(table_id, cluster) - table2 = self._makeOne(table_id, cluster) - comparison_val = (table1 != table2) - self.assertFalse(comparison_val) - - def test___ne__(self): - table1 = self._makeOne('table_id1', 'cluster1') - table2 = self._makeOne('table_id2', 'cluster2') - self.assertNotEqual(table1, table2) - - def _create_test_helper(self, initial_split_keys): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - request_pb = messages_pb2.CreateTableRequest( - initial_split_keys=initial_split_keys, - name=cluster_name, - table_id=TABLE_ID, - ) - - # Create response_pb - response_pb = data_pb2.Table() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # create() has no return value. - - # Perform the method and check the result. - timeout_seconds = 150 - result = table.create(initial_split_keys=initial_split_keys, - timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'CreateTable', - (request_pb, timeout_seconds), - {}, - )]) - - def test_create(self): - initial_split_keys = None - self._create_test_helper(initial_split_keys) - - def test_create_with_split_keys(self): - initial_split_keys = ['s1', 's2'] - self._create_test_helper(initial_split_keys) - - def test_rename(self): - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - - new_table_id = 'new_table_id' - self.assertNotEqual(new_table_id, TABLE_ID) - - client = _Client() - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.RenameTableRequest( - name=table_name, - new_id=new_table_id, - ) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # rename() has no return value. - - # Perform the method and check the result. - timeout_seconds = 97 - result = table.rename(new_table_id, timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'RenameTable', - (request_pb, timeout_seconds), - {}, - )]) - - def test_delete(self): - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._generated import empty_pb2 - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.DeleteTableRequest(name=table_name) - - # Create response_pb - response_pb = empty_pb2.Empty() - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = None # delete() has no return value. - - # Perform the method and check the result. - timeout_seconds = 871 - result = table.delete(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'DeleteTable', - (request_pb, timeout_seconds), - {}, - )]) - - def _list_column_families_helper(self, column_family_name=None): - from gcloud_bigtable._generated import ( - bigtable_table_data_pb2 as data_pb2) - from gcloud_bigtable._generated import ( - bigtable_table_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable.column_family import ColumnFamily - - client = _Client() - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.GetTableRequest(name=table_name) - - # Create response_pb - column_family_id = 'foo' - if column_family_name is None: - column_family_name = (table_name + '/columnFamilies/' + - column_family_id) - column_family = data_pb2.ColumnFamily(name=column_family_name) - response_pb = data_pb2.Table( - column_families={column_family_id: column_family}, - ) - - # Patch the stub used by the API method. - client._table_stub = stub = StubMock(response_pb) - - # Create expected_result. - expected_result = { - column_family_id: ColumnFamily(column_family_id, table), - } - - # Perform the method and check the result. - timeout_seconds = 502 - result = table.list_column_families(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'GetTable', - (request_pb, timeout_seconds), - {}, - )]) - - def test_list_column_families(self): - self._list_column_families_helper() - - def test_list_column_families_failure(self): - column_family_name = 'not-the-right-format' - with self.assertRaises(ValueError): - self._list_column_families_helper( - column_family_name=column_family_name) - - def _read_row_helper(self, chunks): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.row_data import PartialRowData - from gcloud_bigtable import table as MUT - - client = _Client() - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - request_pb = object() # Returned by our mock. - mock_create_row_request = _MockCalled(request_pb) - - # Create response_iterator - row_key = b'row-key' - response_pb = messages_pb2.ReadRowsResponse(row_key=row_key, - chunks=chunks) - response_iterator = [response_pb] - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_iterator) - - # Create expected_result. - if chunks: - expected_result = PartialRowData(row_key) - expected_result._committed = True - expected_result._chunks_encountered = True - else: - expected_result = None - - # Perform the method and check the result. - filter_obj = object() - timeout_seconds = 596 - with _Monkey(MUT, _create_row_request=mock_create_row_request): - result = table.read_row(row_key, filter_=filter_obj, - timeout_seconds=timeout_seconds) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ReadRows', - (request_pb, timeout_seconds), - {}, - )]) - mock_create_row_request.check_called( - self, [(table.name,)], - [{'row_key': row_key, 'filter_': filter_obj}]) - - def test_read_row(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - chunk = messages_pb2.ReadRowsResponse.Chunk(commit_row=True) - chunks = [chunk] - self._read_row_helper(chunks) - - def test_read_empty_row(self): - chunks = [] - self._read_row_helper(chunks) - - def test_read_row_still_partial(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - # There is never a "commit row". - chunk = messages_pb2.ReadRowsResponse.Chunk(reset_row=True) - chunks = [chunk] - with self.assertRaises(ValueError): - self._read_row_helper(chunks) - - def test_read_rows(self): - from gcloud_bigtable._grpc_mocks import StubMock - from gcloud_bigtable._testing import _MockCalled - from gcloud_bigtable._testing import _Monkey - from gcloud_bigtable.row_data import PartialRowsData - from gcloud_bigtable import table as MUT - - client = _Client() - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - request_pb = object() # Returned by our mock. - mock_create_row_request = _MockCalled(request_pb) - - # Create response_iterator - response_iterator = object() - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_iterator) - - # Create expected_result. - expected_result = PartialRowsData(response_iterator) - - # Perform the method and check the result. - start_key = b'start-key' - end_key = b'end-key' - filter_obj = object() - allow_row_interleaving = True - limit = 22 - timeout_seconds = 1111 - with _Monkey(MUT, _create_row_request=mock_create_row_request): - result = table.read_rows( - start_key=start_key, end_key=end_key, filter_=filter_obj, - allow_row_interleaving=allow_row_interleaving, limit=limit, - timeout_seconds=timeout_seconds) - - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'ReadRows', - (request_pb, timeout_seconds), - {}, - )]) - created_kwargs = { - 'start_key': start_key, - 'end_key': end_key, - 'filter_': filter_obj, - 'allow_row_interleaving': allow_row_interleaving, - 'limit': limit, - } - mock_create_row_request.check_called(self, [(table.name,)], - [created_kwargs]) - - def test_sample_row_keys(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable._grpc_mocks import StubMock - - client = _Client() - cluster_name = ('projects/' + PROJECT_ID + '/zones/' + ZONE + - '/clusters/' + CLUSTER_ID) - cluster = _Cluster(cluster_name, client=client) - table = self._makeOne(TABLE_ID, cluster) - - # Create request_pb - table_name = cluster_name + '/tables/' + TABLE_ID - request_pb = messages_pb2.SampleRowKeysRequest(table_name=table_name) - - # Create response_iterator - response_iterator = object() # Just passed to a mock. - - # Patch the stub used by the API method. - client._data_stub = stub = StubMock(response_iterator) - - # Create expected_result. - expected_result = response_iterator - - # Perform the method and check the result. - timeout_seconds = 1333 - result = table.sample_row_keys(timeout_seconds=timeout_seconds) - self.assertEqual(result, expected_result) - self.assertEqual(stub.method_calls, [( - 'SampleRowKeys', - (request_pb, timeout_seconds), - {}, - )]) - - -class Test__create_row_request(unittest2.TestCase): - - def _callFUT(self, table_name, row_key=None, start_key=None, end_key=None, - filter_=None, allow_row_interleaving=None, limit=None): - from gcloud_bigtable.table import _create_row_request - return _create_row_request( - table_name, row_key=row_key, start_key=start_key, end_key=end_key, - filter_=filter_, allow_row_interleaving=allow_row_interleaving, - limit=limit) - - def test_table_name_only(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - result = self._callFUT(table_name) - expected_result = messages_pb2.ReadRowsRequest(table_name=table_name) - self.assertEqual(result, expected_result) - - def test_row_key_row_range_conflict(self): - with self.assertRaises(ValueError): - self._callFUT(None, row_key=object(), end_key=object()) - - def test_row_key(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - row_key = b'row_key' - result = self._callFUT(table_name, row_key=row_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_key=row_key, - ) - self.assertEqual(result, expected_result) - - def test_row_range_start_key(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - start_key = b'start_key' - result = self._callFUT(table_name, start_key=start_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_range=data_pb2.RowRange(start_key=start_key), - ) - self.assertEqual(result, expected_result) - - def test_row_range_end_key(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - end_key = b'end_key' - result = self._callFUT(table_name, end_key=end_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_range=data_pb2.RowRange(end_key=end_key), - ) - self.assertEqual(result, expected_result) - - def test_row_range_both_keys(self): - from gcloud_bigtable._generated import bigtable_data_pb2 as data_pb2 - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - start_key = b'start_key' - end_key = b'end_key' - result = self._callFUT(table_name, start_key=start_key, - end_key=end_key) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - row_range=data_pb2.RowRange(start_key=start_key, end_key=end_key), - ) - self.assertEqual(result, expected_result) - - def test_with_filter(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - from gcloud_bigtable.row import RowFilter - - table_name = 'table_name' - row_filter = RowFilter(row_sample_filter=0.33) - result = self._callFUT(table_name, filter_=row_filter) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - filter=row_filter.to_pb(), - ) - self.assertEqual(result, expected_result) - - def test_with_allow_row_interleaving(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - allow_row_interleaving = True - result = self._callFUT(table_name, - allow_row_interleaving=allow_row_interleaving) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - allow_row_interleaving=allow_row_interleaving, - ) - self.assertEqual(result, expected_result) - - def test_with_limit(self): - from gcloud_bigtable._generated import ( - bigtable_service_messages_pb2 as messages_pb2) - - table_name = 'table_name' - limit = 1337 - result = self._callFUT(table_name, limit=limit) - expected_result = messages_pb2.ReadRowsRequest( - table_name=table_name, - num_rows_limit=limit, - ) - self.assertEqual(result, expected_result) - - -class _Client(object): - - data_stub = None - cluster_stub = None - operations_stub = None - table_stub = None - - -class _Cluster(object): - - def __init__(self, name, client=None, timeout_seconds=None): - self.name = name - self.client = client - self.timeout_seconds = timeout_seconds diff --git a/pylintrc_default b/pylintrc_default deleted file mode 100644 index 1be8524..0000000 --- a/pylintrc_default +++ /dev/null @@ -1,418 +0,0 @@ -# PyLint config for 'gcloud' *library* code. -# -# NOTES: -# -# - Rules for test / demo code are generated into 'pylintrc_reduced' -# as deltas from this configuration by the 'run_pylint.py' script. -# -# - 'RATIONALE: API mapping' as a defense for non-default settings is -# based on the fact that this library maps APIs which are outside our -# control, and adhereing to the out-of-the-box defaults would induce -# breakage / complexity in those mappings -# -[MASTER] - -# Specify a configuration file. -# DEFAULT: rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -# DEFAULT: init-hook= - -# Profiled execution. -# DEFAULT: profile=no - -# Add files or directories to the blacklist. They should be base names, not -# paths. -# DEFAULT: ignore=CVS -# NOTE: This path must be relative due to the use of -# os.walk in astroid.modutils.get_module_files. -# RATIONALE: -# *_pb2.py: protobuf-generated code. -ignore = - annotations_pb2.py, - any_pb2.py, - bigtable_cluster_data_pb2.py, - bigtable_cluster_service_messages_pb2.py, - bigtable_cluster_service_pb2.py, - bigtable_data_pb2.py, - bigtable_service_messages_pb2.py, - bigtable_service_pb2.py, - bigtable_table_data_pb2.py, - bigtable_table_service_messages_pb2.py, - bigtable_table_service_pb2.py, - duration_pb2.py, - empty_pb2.py, - http_pb2.py, - operations_pb2.py, - status_pb2.py, - timestamp_pb2.py, - -# Pickle collected data for later comparisons. -# DEFAULT: persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -# DEFAULT: load-plugins= -# RATIONALE: We want to make sure our docstrings match the objects -# they document. -load-plugins=pylint.extensions.check_docs - -# DEPRECATED -# DEFAULT: include-ids=no - -# DEPRECATED -# DEFAULT: symbols=no - - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -# DEFAULT: enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -# DEFAULT: disable= -# RATIONALE: -# - maybe-no-member: bi-modal functions confuse pylint type inference. -# - no-member: indirections in protobuf-generated code -# - protected-access: helpers use '_foo' of classes from generated code. -# - redefined-builtin: use of 'id', 'type', 'filter' args in API-bound funcs; -# use of 'NotImplemented' to map HTTP response code. -# - similarities: 'Bucket' and 'Blob' define 'metageneration' and 'owner' with -# identical implementation but different docstrings. -# - star-args: standard Python idioms for varargs: -# ancestor = Query().filter(*order_props) -# - method-hidden: Decorating a method in a class (e.g. in _DefaultsContainer) -# @_lazy_property_deco -# def dataset_id(): -# ... -disable = - maybe-no-member, - no-member, - protected-access, - redefined-builtin, - similarities, - star-args, - method-hidden, - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -# DEFAULT: output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -# DEFAULT: files-output=no - -# Tells whether to display a full report or only the messages -# DEFAULT: reports=yes -# RATIONALE: run from Travis / tox, and don't need / want to parse output. -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -# DEFAULT: evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (RP0004). -# DEFAULT: comment=no - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -# DEFAULT: min-similarity-lines=4 - -# Ignore comments when computing similarities. -# DEFAULT: ignore-comments=yes - -# Ignore docstrings when computing similarities. -# DEFAULT: ignore-docstrings=yes - -# Ignore imports when computing similarities. -# DEFAULT: ignore-imports=no - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -# DEFAULT: init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -# DEFAULT: dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -# DEFAULT: additional-builtins= - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -# DEFAULT: logging-modules=logging - - -[FORMAT] - -# Maximum number of characters on a single line. -# DEFAULT: max-line-length=80 - -# Regexp for a line that is allowed to be longer than the limit. -# DEFAULT: ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -# DEFAULT: single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled -# DEFAULT: no-space-check=trailing-comma,dict-separator -# RATIONALE: pylint ignores whitespace checks around the -# constructs "dict-separator" (cases like {1:2}) and -# "trailing-comma" (cases like {1: 2, }). -# By setting "no-space-check" to empty whitespace checks will be -# enforced around both constructs. -no-space-check = - -# Maximum number of lines in a module -# DEFAULT: max-module-lines=1000 -# RATIONALE: API-mapping -max-module-lines=1500 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -# DEFAULT: indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -# DEFAULT: indent-after-paren=4 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -# DEFAULT: notes=FIXME,XXX,TODO - - -[BASIC] - -# Required attributes for module, separated by a comma -# DEFAULT: required-attributes= - -# List of builtins function names that should not be used, separated by a comma -# DEFAULT: bad-functions=map,filter,apply,input,file - -# Good variable names which should always be accepted, separated by a comma -# DEFAULT: good-names=i,j,k,ex,Run,_ -# RATIONALE: 'pb' and 'id' have well-understood meainings in the code. -good-names = i, j, k, ex, Run, _, - pb, - id, - -# Bad variable names which should always be refused, separated by a comma -# DEFAULT: bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -# DEFAULT: name-group= - -# Include a hint for the correct naming format with invalid-name -# DEFAULT: include-naming-hint=no - -# Regular expression matching correct function names -# DEFAULT: function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -# DEFAULT: function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -# DEFAULT: variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -# DEFAULT: variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names -# DEFAULT: const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -# DEFAULT: const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -# DEFAULT: attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -# DEFAULT: attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names -# DEFAULT: argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -# DEFAULT: argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -# DEFAULT: class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -# DEFAULT: class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -# DEFAULT: inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -# DEFAULT: inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -# DEFAULT: class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -# DEFAULT: class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -# DEFAULT: module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -# DEFAULT: module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -# DEFAULT: method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -# DEFAULT: method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -# DEFAULT: no-docstring-rgx=__.*__ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -# DEFAULT: docstring-min-length=-1 - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -# DEFAULT: ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis -# DEFAULT: ignored-modules= - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -# DEFAULT: ignored-classes=SQLObject - -# When zope mode is activated, add a predefined set of Zope acquired attributes -# to generated-members. -# DEFAULT: zope=no - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -# DEFAULT: generated-members=REQUEST,acl_users,aq_parent - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -# DEFAULT: deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -# DEFAULT: import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -# DEFAULT: ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -# DEFAULT: int-import-graph= - - -[CLASSES] - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -# DEFAULT: ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -# DEFAULT: defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -# DEFAULT: valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -# DEFAULT: valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -# DEFAULT: max-args=5 -# RATIONALE: RowFilter constructor -max-args = 13 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -# DEFAULT: ignored-argument-names=_.* - -# Maximum number of locals for function / method body -# DEFAULT: max-locals=15 -max-locals=20 - -# Maximum number of return / yield for function / method body -# DEFAULT: max-returns=6 - -# Maximum number of branch for function / method body -# DEFAULT: max-branches=12 - -# Maximum number of statements in function / method body -# DEFAULT: max-statements=50 - -# Maximum number of parents for a class (see R0901). -# DEFAULT: max-parents=7 - -# Maximum number of attributes for a class (see R0902). -# DEFAULT: max-attributes=7 -# RATIONALE: API mapping -max-attributes=15 - -# Minimum number of public methods for a class (see R0903). -# DEFAULT: min-public-methods=2 -# RATIONALE: context mgrs may have *no* public methods -min-public-methods=0 - -# Maximum number of public methods for a class (see R0904). -# DEFAULT: max-public-methods=20 -# RATIONALE: API mapping -max-public-methods=40 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -# DEFAULT: overgeneral-exceptions=Exception diff --git a/run_pylint.py b/run_pylint.py deleted file mode 100644 index 99b0647..0000000 --- a/run_pylint.py +++ /dev/null @@ -1,237 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Custom script to run PyLint on gcloud-python-bigtable codebase. - -This runs pylint as a script via subprocess in two different -subprocesses. The first lints the production/library code -using the default rc file (PRODUCTION_RC). The second lints the -demo/test code using an rc file (TEST_RC) which allows more style -violations (hence it has a reduced number of style checks). -""" - -import ConfigParser -import copy -import os -import subprocess -import sys - - -IGNORED_DIRECTORIES = [ - 'gcloud_bigtable/_generated', -] -IGNORED_FILES = [ - 'docs/conf.py', - 'setup.py', -] -PRODUCTION_RC = 'pylintrc_default' -TEST_RC = 'pylintrc_reduced' -TEST_DISABLED_MESSAGES = [ - 'attribute-defined-outside-init', - 'exec-used', - 'import-error', - 'invalid-name', - 'missing-docstring', - 'no-init', - 'no-self-use', - 'superfluous-parens', - 'too-few-public-methods', - 'too-many-locals', - 'too-many-public-methods', - 'unbalanced-tuple-unpacking', -] -TEST_RC_ADDITIONS = { - 'MESSAGES CONTROL': { - 'disable': ', '.join(TEST_DISABLED_MESSAGES), - }, -} - - -def read_config(filename): - """Reads pylintrc config onto native ConfigParser object.""" - config = ConfigParser.ConfigParser() - with open(filename, 'r') as file_obj: - config.readfp(file_obj) - return config - - -def make_test_rc(base_rc_filename, additions_dict, target_filename): - """Combines a base rc and test additions into single file.""" - main_cfg = read_config(base_rc_filename) - - # Create fresh config for test, which must extend production. - test_cfg = ConfigParser.ConfigParser() - test_cfg._sections = copy.deepcopy(main_cfg._sections) - - for section, opts in additions_dict.items(): - curr_section = test_cfg._sections.setdefault( - section, test_cfg._dict()) - for opt, opt_val in opts.items(): - curr_val = curr_section.get(opt) - if curr_val is None: - raise KeyError('Expected to be adding to existing option.') - curr_val = curr_val.rstrip(',') - curr_section[opt] = '%s, %s' % (curr_val, opt_val) - - with open(target_filename, 'w') as file_obj: - test_cfg.write(file_obj) - - -def valid_filename(filename): - """Checks if a file is a Python file and is not ignored.""" - for directory in IGNORED_DIRECTORIES: - if filename.startswith(directory): - return False - return (filename.endswith('.py') and - filename not in IGNORED_FILES) - - -def is_production_filename(filename): - """Checks if the file contains production code. - - :rtype: bool - :returns: Boolean indicating production status. - """ - return not ('demo' in filename or 'test' in filename) - - -def get_files_for_linting(allow_limited=True): - """Gets a list of files in the repository. - - By default, returns all files via ``git ls-files``. However, in some cases - uses a specific commit or branch (a so-called diff base) to compare - against for changed files. (This requires ``allow_limited=True``.) - - To speed up linting on Travis pull requests against master, we manually - set the diff base to origin/master. We don't do this on non-pull requests - since origin/master will be equivalent to the currently checked out code. - One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base but - this value is not dependable. - - To allow faster local ``tox`` runs, the environment variables - ``GCLOUD_REMOTE_FOR_LINT`` and ``GCLOUD_BRANCH_FOR_LINT`` can be set to - specify a remote branch to diff against. - - :type allow_limited: bool - :param allow_limited: Boolean indicating if a reduced set of files can - be used. - - :rtype: pair - :returns: Tuple of the diff base using the the list of filenames to be - linted. - """ - diff_base = None - if (os.getenv('TRAVIS_BRANCH') == 'master' and - os.getenv('TRAVIS_PULL_REQUEST') != 'false'): - # In the case of a pull request into master, we want to - # diff against HEAD in master. - diff_base = 'origin/master' - elif os.getenv('TRAVIS') is None: - # Only allow specified remote and branch in local dev. - remote = os.getenv('GCLOUD_REMOTE_FOR_LINT') - branch = os.getenv('GCLOUD_BRANCH_FOR_LINT') - if remote is not None and branch is not None: - diff_base = '%s/%s' % (remote, branch) - - if diff_base is not None and allow_limited: - result = subprocess.check_output(['git', 'diff', '--name-only', - diff_base]) - print 'Using files changed relative to %s:' % (diff_base,) - print '-' * 60 - print result.rstrip('\n') # Don't print trailing newlines. - print '-' * 60 - else: - print 'Diff base not specified, listing all files in repository.' - result = subprocess.check_output(['git', 'ls-files']) - - return result.rstrip('\n').split('\n'), diff_base - - -def get_python_files(all_files=None): - """Gets a list of all Python files in the repository that need linting. - - Relies on :func:`get_files_for_linting()` to determine which files should - be considered. - - NOTE: This requires ``git`` to be installed and requires that this - is run within the ``git`` repository. - - :type all_files: list or :data:`NoneType ` - :param all_files: Optional list of files to be linted. - - :rtype: tuple - :returns: A tuple containing two lists and a boolean. The first list - contains all production files, the next all test/demo files and - the boolean indicates if a restricted fileset was used. - """ - using_restricted = False - if all_files is None: - all_files, diff_base = get_files_for_linting() - using_restricted = diff_base is not None - - library_files = [] - non_library_files = [] - for filename in all_files: - if valid_filename(filename): - if is_production_filename(filename): - library_files.append(filename) - else: - non_library_files.append(filename) - - return library_files, non_library_files, using_restricted - - -def lint_fileset(filenames, rcfile, description): - """Lints a group of files using a given rcfile.""" - # Only lint filenames that exist. For example, 'git diff --name-only' - # could spit out deleted / renamed files. Another alternative could - # be to use 'git diff --name-status' and filter out files with a - # status of 'D'. - filenames = [filename for filename in filenames - if os.path.exists(filename)] - if filenames: - rc_flag = '--rcfile=%s' % (rcfile,) - pylint_shell_command = ['pylint', rc_flag] + filenames - status_code = subprocess.call(pylint_shell_command) - if status_code != 0: - error_message = ('Pylint failed on %s with ' - 'status %d.' % (description, status_code)) - print >> sys.stderr, error_message - sys.exit(status_code) - else: - print 'Skipping %s, no files to lint.' % (description,) - - -def main(): - """Script entry point. Lints both sets of files.""" - make_test_rc(PRODUCTION_RC, TEST_RC_ADDITIONS, TEST_RC) - library_files, non_library_files, using_restricted = get_python_files() - try: - lint_fileset(library_files, PRODUCTION_RC, 'library code') - lint_fileset(non_library_files, TEST_RC, 'test and demo code') - except SystemExit: - if not using_restricted: - raise - - message = 'Restricted lint failed, expanding to full fileset.' - print >> sys.stderr, message - all_files, _ = get_files_for_linting(allow_limited=False) - library_files, non_library_files, _ = get_python_files( - all_files=all_files) - lint_fileset(library_files, PRODUCTION_RC, 'library code') - lint_fileset(non_library_files, TEST_RC, 'test and demo code') - - -if __name__ == '__main__': - main() diff --git a/scripts/check_generate.py b/scripts/check_generate.py deleted file mode 100644 index 76c9cd3..0000000 --- a/scripts/check_generate.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Checking that protobuf generated modules import correctly.""" - -from __future__ import print_function - -import glob -import os - - -def main(): - """Import all PB2 files.""" - print('>>> import gcloud_bigtable._generated') - _ = __import__('gcloud_bigtable._generated') - pb2_files = sorted(glob.glob('gcloud_bigtable/_generated/*pb2.py')) - for filename in pb2_files: - basename = os.path.basename(filename) - module_name, _ = os.path.splitext(basename) - - print('>>> from gcloud_bigtable._generated import ' + module_name) - _ = __import__('gcloud_bigtable._generated', fromlist=[module_name]) - - -if __name__ == '__main__': - main() diff --git a/scripts/nose_with_env.sh b/scripts/nose_with_env.sh deleted file mode 100755 index d83e01d..0000000 --- a/scripts/nose_with_env.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ev - -if [[ -n "$(which brew)" ]]; then - export LD_LIBRARY_PATH=$(brew --prefix)/lib -fi - -nosetests ${@} diff --git a/scripts/pep8_on_repo.sh b/scripts/pep8_on_repo.sh deleted file mode 100755 index 6afcd7f..0000000 --- a/scripts/pep8_on_repo.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ev - -pep8 $(git ls-files '*py') diff --git a/scripts/rewrite_imports.py b/scripts/rewrite_imports.py deleted file mode 100644 index 7685411..0000000 --- a/scripts/rewrite_imports.py +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Build script for rewriting imports for protobuf generated modules. - -Intended to be used for Google Cloud Bigtable protos (google/bigtable/v1) -and the dependent modules (google/api and google/protobuf). -""" - -import glob - - -IMPORT_TEMPLATE = 'import %s' -IMPORT_FROM_TEMPLATE = 'from %s import ' -PROTOBUF_IMPORT_TEMPLATE = 'from google.protobuf import %s ' -REPLACE_PROTOBUF_IMPORT_TEMPLATE = 'from gcloud_bigtable._generated import %s ' -REPLACEMENTS = { - 'google.api': 'gcloud_bigtable._generated', - 'google.bigtable.admin.cluster.v1': 'gcloud_bigtable._generated', - 'google.bigtable.admin.table.v1': 'gcloud_bigtable._generated', - 'google.bigtable.v1': 'gcloud_bigtable._generated', - 'google.longrunning': 'gcloud_bigtable._generated', - 'google.rpc': 'gcloud_bigtable._generated', -} -GOOGLE_PROTOBUF_CUSTOM = ( - 'any_pb2', - 'duration_pb2', - 'empty_pb2', - 'timestamp_pb2', -) - - -def transform_old_to_new(line, old_module, new_module, - ignore_import_from=False): - """Transforms from an old module to a new one. - - First checks if a line starts with - "from {old_module} import ..." - then checks if the line contains - "import {old_module} ..." - and finally checks if the line starts with (ignoring whitespace) - "{old_module} ..." - - In any of these cases, "{old_module}" is replaced with "{new_module}". - If none match, nothing is returned. - - :type line: str - :param line: The line to be transformed. - - :type old_module: str - :param old_module: The import to be re-written. - - :type new_module: str - :param new_module: The new location of the re-written import. - - :type ignore_import_from: bool - :param ignore_import_from: Flag to determine if the "from * import" - statements should be ignored. - - :rtype: :class:`str` or :data:`NoneType ` - :returns: The transformed line if the old module was found, otherwise - does nothing. - """ - if not ignore_import_from: - import_from_statement = IMPORT_FROM_TEMPLATE % (old_module,) - if line.startswith(import_from_statement): - new_import_from_statement = IMPORT_FROM_TEMPLATE % (new_module,) - # Only replace the first instance of the import statement. - return line.replace(import_from_statement, - new_import_from_statement, 1) - - # If the line doesn't start with a "from * import *" statement, it - # may still contain a "import * ..." statement. - import_statement = IMPORT_TEMPLATE % (old_module,) - if import_statement in line: - new_import_statement = IMPORT_TEMPLATE % (new_module,) - # Only replace the first instance of the import statement. - return line.replace(import_statement, - new_import_statement, 1) - - # Also need to change references to the standalone imports. As a - # stop-gap we fix references to them at the beginning of a line - # (ignoring whitespace). - if line.lstrip().startswith(old_module): - # Only replace the first instance of the old_module. - return line.replace(old_module, new_module, 1) - - -def transform_line(line): - """Transforms an import line in a PB2 module. - - If the line is not an import of one of the packages in - ``REPLACEMENTS`` or ``GOOGLE_PROTOBUF_CUSTOM``, does nothing and returns - the original. Otherwise it replaces the package matched with our local - package or directly rewrites the custom ``google.protobuf`` import - statement. - - :type line: str - :param line: The line to be transformed. - - :rtype: str - :returns: The transformed line. - """ - for old_module, new_module in REPLACEMENTS.iteritems(): - result = transform_old_to_new(line, old_module, new_module) - if result is not None: - return result - - for custom_protobuf_module in GOOGLE_PROTOBUF_CUSTOM: - # We don't use the "from * import" check in transform_old_to_new - # because part of `google.protobuf` comes from the installed - # `protobuf` library. - import_from_statement = PROTOBUF_IMPORT_TEMPLATE % ( - custom_protobuf_module,) - if line.startswith(import_from_statement): - new_import_from_statement = REPLACE_PROTOBUF_IMPORT_TEMPLATE % ( - custom_protobuf_module,) - # Only replace the first instance of the import statement. - return line.replace(import_from_statement, - new_import_from_statement, 1) - - old_module = 'google.protobuf.' + custom_protobuf_module - new_module = 'gcloud_bigtable._generated.' + custom_protobuf_module - result = transform_old_to_new(line, old_module, new_module, - ignore_import_from=True) - if result is not None: - return result - - # If no matches, there is nothing to transform. - return line - - -def rewrite_file(filename): - """Rewrites a given PB2 modules. - - :type filename: str - :param filename: The name of the file to be rewritten. - """ - with open(filename, 'rU') as file_obj: - content_lines = file_obj.read().split('\n') - - new_content = [] - for line in content_lines: - new_content.append(transform_line(line)) - - with open(filename, 'w') as file_obj: - file_obj.write('\n'.join(new_content)) - - -def main(): - """Rewrites all PB2 files.""" - pb2_files = glob.glob('gcloud_bigtable/_generated/*pb2.py') - for filename in pb2_files: - rewrite_file(filename) - - -if __name__ == '__main__': - main() diff --git a/scripts/sphinx_with_env.sh b/scripts/sphinx_with_env.sh deleted file mode 100755 index f2ff46f..0000000 --- a/scripts/sphinx_with_env.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ev - -if [[ -n "$(which brew)" ]]; then - export LD_LIBRARY_PATH=$(brew --prefix)/lib -fi - -sphinx-build ${@} diff --git a/setup.py b/setup.py deleted file mode 100644 index b8c31da..0000000 --- a/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -import os - -from setuptools import setup -from setuptools import find_packages - - -HERE = os.path.abspath(os.path.dirname(__file__)) -with open(os.path.join(HERE, 'README.md')) as file_obj: - README = file_obj.read() - - -REQUIREMENTS = [ - 'httplib2 >= 0.9.1', - 'oauth2client >= 1.4.6', - 'protobuf >= 3.0.0a3', - 'pytz', - 'six >= 1.6.1', -] - -setup( - name='gcloud-bigtable', - version='0.0.1', - description='API Client library for Google Cloud Bigtable', - author='Google Cloud Platform', - author_email='daniel.j.hermes@gmail.com', - long_description=README, - scripts=[], - url='https://github.com/dhermes/gcloud-python-bigtable', - packages=find_packages(), - license='Apache 2.0', - platforms='Posix; MacOS X; Windows', - include_package_data=True, - zip_safe=False, - install_requires=REQUIREMENTS, - classifiers=[ - 'Development Status :: 1 - Planning', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: Apache Software License', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Internet', - ] -) diff --git a/system_tests/local_setup.sample b/system_tests/local_setup.sample deleted file mode 100644 index b8a47a4..0000000 --- a/system_tests/local_setup.sample +++ /dev/null @@ -1,3 +0,0 @@ -export GCLOUD_REMOTE_FOR_LINT="upstream" -export GCLOUD_BRANCH_FOR_LINT="master" -export GCLOUD_TESTS_PROJECT_ID=some-project-id diff --git a/system_tests/run.py b/system_tests/run.py deleted file mode 100644 index 2755f37..0000000 --- a/system_tests/run.py +++ /dev/null @@ -1,412 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import datetime -import operator -import os -import pytz -import time -import unittest2 - -from oauth2client.client import GoogleCredentials - -from gcloud_bigtable._helpers import _microseconds_to_timestamp -from gcloud_bigtable._helpers import _timestamp_to_microseconds -from gcloud_bigtable.client import Client -from gcloud_bigtable.column_family import GarbageCollectionRule -from gcloud_bigtable.row import RowFilter -from gcloud_bigtable.row import RowFilterChain -from gcloud_bigtable.row import RowFilterUnion -from gcloud_bigtable.row_data import Cell -from gcloud_bigtable.row_data import PartialRowData - - -PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID') -CENTRAL_1C_ZONE = 'us-central1-c' -NOW_MILLIS = int(1000 * time.time()) -CLUSTER_ID = 'gcloud-python-%d' % (NOW_MILLIS,) -SERVE_NODES = 3 -TABLE_ID = 'gcloud-python-test-table' -COLUMN_FAMILY_ID1 = u'col-fam-id1' -COLUMN_FAMILY_ID2 = u'col-fam-id2' -COL_NAME1 = b'col-name1' -COL_NAME2 = b'col-name2' -COL_NAME3 = b'col-name3-but-other-fam' -CELL_VAL1 = b'cell-val' -CELL_VAL2 = b'cell-val-newer' -CELL_VAL3 = b'altcol-cell-val' -CELL_VAL4 = b'foo' -ROW_KEY = b'row-key' -ROW_KEY_ALT = b'row-key-alt' -EXPECTED_ZONES = ( - 'asia-east1-b', - 'europe-west1-c', - 'us-central1-b', - CENTRAL_1C_ZONE, -) -EXISTING_CLUSTERS = [] -CREDENTIALS = GoogleCredentials.get_application_default() -CLIENT = Client(project=PROJECT_ID, credentials=CREDENTIALS, admin=True) -CLUSTER = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID, - display_name=CLUSTER_ID) - - -def setUpModule(): - CLIENT.start() - clusters, failed_zones = CLIENT.list_clusters() - - if len(failed_zones) != 0: - raise ValueError('List clusters failed in module set up.') - - EXISTING_CLUSTERS[:] = clusters - - # After listing, create the test cluster. - CLUSTER.create() - - -def tearDownModule(): - CLUSTER.delete() - CLIENT.stop() - - -class TestClusterAdminAPI(unittest2.TestCase): - - def setUp(self): - self.clusters_to_delete = [] - - def tearDown(self): - for cluster in self.clusters_to_delete: - cluster.delete() - - def test_list_zones(self): - zones = CLIENT.list_zones() - self.assertEqual(sorted(zones), list(EXPECTED_ZONES)) - - def test_list_clusters(self): - clusters, failed_zones = CLIENT.list_clusters() - self.assertEqual(failed_zones, []) - # We have added one new cluster in `setUpModule`. - self.assertEqual(len(clusters), len(EXISTING_CLUSTERS) + 1) - for cluster in clusters: - cluster_existence = (cluster in EXISTING_CLUSTERS or - cluster == CLUSTER) - self.assertTrue(cluster_existence) - - def test_reload(self): - # Use same arguments as CLUSTER (created in `setUpModule`). - cluster = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID) - # Make sure metadata unset before reloading. - cluster.display_name = None - cluster.serve_nodes = None - - cluster.reload() - self.assertEqual(cluster.display_name, CLUSTER.display_name) - self.assertEqual(cluster.serve_nodes, CLUSTER.serve_nodes) - - def test_create_cluster(self): - cluster_id = '%s-a' % (CLUSTER_ID,) - cluster = CLIENT.cluster(CENTRAL_1C_ZONE, cluster_id) - cluster.create() - # Make sure this cluster gets deleted after the test case. - self.clusters_to_delete.append(cluster) - - # We want to make sure the operation completes. - time.sleep(2) - self.assertTrue(cluster.operation_finished()) - - # Create a new cluster instance and make sure it is the same. - cluster_alt = CLIENT.cluster(CENTRAL_1C_ZONE, cluster_id) - cluster_alt.reload() - - self.assertEqual(cluster, cluster_alt) - self.assertEqual(cluster.display_name, cluster_alt.display_name) - self.assertEqual(cluster.serve_nodes, cluster_alt.serve_nodes) - - def test_update(self): - curr_display_name = CLUSTER.display_name - CLUSTER.display_name = 'Foo Bar Baz' - CLUSTER.update() - - # We want to make sure the operation completes. - time.sleep(2) - self.assertTrue(CLUSTER.operation_finished()) - - # Create a new cluster instance and make sure it is the same. - cluster_alt = CLIENT.cluster(CENTRAL_1C_ZONE, CLUSTER_ID) - self.assertNotEqual(cluster_alt.display_name, - CLUSTER.display_name) - cluster_alt.reload() - self.assertEqual(cluster_alt.display_name, - CLUSTER.display_name) - - # Make sure to put the cluster back the way it was for the - # other test cases. - CLUSTER.display_name = curr_display_name - CLUSTER.update() - - # We want to make sure the operation completes. - time.sleep(2) - self.assertTrue(CLUSTER.operation_finished()) - - -class TestTableAdminAPI(unittest2.TestCase): - - @classmethod - def setUpClass(self): - self._table = CLUSTER.table(TABLE_ID) - self._table.create() - - @classmethod - def tearDownClass(self): - self._table.delete() - - def setUp(self): - self.tables_to_delete = [] - - def tearDown(self): - for table in self.tables_to_delete: - table.delete() - - def test_list_tables(self): - # Since `CLUSTER` is newly created in `setUpModule`, the table - # created in `setUpClass` here will be the only one. - tables = CLUSTER.list_tables() - self.assertEqual(tables, [self._table]) - - def test_create_table(self): - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - # First, create a sorted version of our expected result. - name_attr = operator.attrgetter('name') - expected_tables = sorted([temp_table, self._table], key=name_attr) - - # Then query for the tables in the cluster and sort them by - # name as well. - tables = CLUSTER.list_tables() - sorted_tables = sorted(tables, key=name_attr) - self.assertEqual(sorted_tables, expected_tables) - - def test_create_column_family(self): - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - self.assertEqual(temp_table.list_column_families(), {}) - gc_rule = GarbageCollectionRule(max_num_versions=1) - column_family = temp_table.column_family(COLUMN_FAMILY_ID1, - gc_rule=gc_rule) - column_family.create() - - col_fams = temp_table.list_column_families() - - self.assertEqual(len(col_fams), 1) - retrieved_col_fam = col_fams[COLUMN_FAMILY_ID1] - self.assertTrue(retrieved_col_fam.table is column_family.table) - self.assertEqual(retrieved_col_fam.column_family_id, - column_family.column_family_id) - self.assertEqual(retrieved_col_fam.gc_rule, gc_rule) - - def test_delete_column_family(self): - temp_table_id = 'foo-bar-baz-table' - temp_table = CLUSTER.table(temp_table_id) - temp_table.create() - self.tables_to_delete.append(temp_table) - - self.assertEqual(temp_table.list_column_families(), {}) - column_family = temp_table.column_family(COLUMN_FAMILY_ID1) - column_family.create() - - # Make sure the family is there before deleting it. - col_fams = temp_table.list_column_families() - self.assertEqual(list(col_fams.keys()), [COLUMN_FAMILY_ID1]) - - column_family.delete() - # Make sure we have successfully deleted it. - self.assertEqual(temp_table.list_column_families(), {}) - - -class TestDataAPI(unittest2.TestCase): - - @classmethod - def setUpClass(self): - self._table = table = CLUSTER.table(TABLE_ID) - table.create() - table.column_family(COLUMN_FAMILY_ID1).create() - table.column_family(COLUMN_FAMILY_ID2).create() - - @classmethod - def tearDownClass(self): - # Will also delete any data contained in the table. - self._table.delete() - - def setUp(self): - self.rows_to_delete = [] - - def tearDown(self): - for row in self.rows_to_delete: - row.clear_mutations() - row.delete() - row.commit() - - def _write_to_row(self, row1=None, row2=None, row3=None, row4=None): - timestamp1 = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) - # Must be millisecond granularity. - timestamp1 = _microseconds_to_timestamp( - _timestamp_to_microseconds(timestamp1)) - # 1000 microseconds is a millisecond - timestamp2 = timestamp1 + datetime.timedelta(microseconds=1000) - timestamp3 = timestamp1 + datetime.timedelta(microseconds=2000) - timestamp4 = timestamp1 + datetime.timedelta(microseconds=3000) - if row1 is not None: - row1.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL1, - timestamp=timestamp1) - if row2 is not None: - row2.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, CELL_VAL2, - timestamp=timestamp2) - if row3 is not None: - row3.set_cell(COLUMN_FAMILY_ID1, COL_NAME2, CELL_VAL3, - timestamp=timestamp3) - if row4 is not None: - row4.set_cell(COLUMN_FAMILY_ID2, COL_NAME3, CELL_VAL4, - timestamp=timestamp4) - - # Create the cells we will check. - cell1 = Cell(CELL_VAL1, timestamp1) - cell2 = Cell(CELL_VAL2, timestamp2) - cell3 = Cell(CELL_VAL3, timestamp3) - cell4 = Cell(CELL_VAL4, timestamp4) - return cell1, cell2, cell3, cell4 - - def test_read_row(self): - row = self._table.row(ROW_KEY) - self.rows_to_delete.append(row) - - cell1, cell2, cell3, cell4 = self._write_to_row(row, row, row, row) - row.commit() - - # Read back the contents of the row. - partial_row_data = self._table.read_row(ROW_KEY) - self.assertTrue(partial_row_data.committed) - self.assertEqual(partial_row_data.row_key, ROW_KEY) - - # Check the cells match. - ts_attr = operator.attrgetter('timestamp') - expected_row_contents = { - COLUMN_FAMILY_ID1: { - COL_NAME1: sorted([cell1, cell2], key=ts_attr, reverse=True), - COL_NAME2: [cell3], - }, - COLUMN_FAMILY_ID2: { - COL_NAME3: [cell4], - }, - } - self.assertEqual(partial_row_data.cells, expected_row_contents) - - def test_read_rows(self): - row = self._table.row(ROW_KEY) - row_alt = self._table.row(ROW_KEY_ALT) - self.rows_to_delete.extend([row, row_alt]) - - cell1, cell2, cell3, cell4 = self._write_to_row(row, row_alt, - row, row_alt) - row.commit() - row_alt.commit() - - rows_data = self._table.read_rows() - self.assertEqual(rows_data.rows, {}) - rows_data.consume_all() - - # NOTE: We should refrain from editing protected data on instances. - # Instead we should make the values public or provide factories - # for constructing objects with them. - row_data = PartialRowData(ROW_KEY) - row_data._chunks_encountered = True - row_data._committed = True - row_data._cells = { - COLUMN_FAMILY_ID1: { - COL_NAME1: [cell1], - COL_NAME2: [cell3], - }, - } - - row_alt_data = PartialRowData(ROW_KEY_ALT) - row_alt_data._chunks_encountered = True - row_alt_data._committed = True - row_alt_data._cells = { - COLUMN_FAMILY_ID1: { - COL_NAME1: [cell2], - }, - COLUMN_FAMILY_ID2: { - COL_NAME3: [cell4], - }, - } - - expected_rows = { - ROW_KEY: row_data, - ROW_KEY_ALT: row_alt_data, - } - self.assertEqual(rows_data.rows, expected_rows) - - def test_read_with_label_applied(self): - row = self._table.row(ROW_KEY) - self.rows_to_delete.append(row) - - cell1, _, cell3, _ = self._write_to_row(row, None, row) - row.commit() - - # Combine a label with column 1. - label1 = u'label-red' - label1_filter = RowFilter(apply_label_transformer=label1) - col1_filter = RowFilter(column_qualifier_regex_filter=COL_NAME1) - chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) - - # Combine a label with column 2. - label2 = u'label-blue' - label2_filter = RowFilter(apply_label_transformer=label2) - col2_filter = RowFilter(column_qualifier_regex_filter=COL_NAME2) - chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) - - # Bring our two labeled columns together. - row_filter = RowFilterUnion(filters=[chain1, chain2]) - partial_row_data = self._table.read_row(ROW_KEY, filter_=row_filter) - self.assertTrue(partial_row_data.committed) - self.assertEqual(partial_row_data.row_key, ROW_KEY) - - cells_returned = partial_row_data.cells - col_fam1 = cells_returned.pop(COLUMN_FAMILY_ID1) - # Make sure COLUMN_FAMILY_ID1 was the only key. - self.assertEqual(len(cells_returned), 0) - - cell1_new, = col_fam1.pop(COL_NAME1) - cell3_new, = col_fam1.pop(COL_NAME2) - # Make sure COL_NAME1 and COL_NAME2 were the only keys. - self.assertEqual(len(col_fam1), 0) - - # Check that cell1 has matching values and gained a label. - self.assertEqual(cell1_new.value, cell1.value) - self.assertEqual(cell1_new.timestamp, cell1.timestamp) - self.assertEqual(cell1.labels, []) - self.assertEqual(cell1_new.labels, [label1]) - - # Check that cell3 has matching values and gained a label. - self.assertEqual(cell3_new.value, cell3.value) - self.assertEqual(cell3_new.timestamp, cell3.timestamp) - self.assertEqual(cell3.labels, []) - self.assertEqual(cell3_new.labels, [label2]) diff --git a/system_tests/run_happybase.py b/system_tests/run_happybase.py deleted file mode 100644 index 082f171..0000000 --- a/system_tests/run_happybase.py +++ /dev/null @@ -1,981 +0,0 @@ -# Copyright 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import operator -import struct -import time -import unittest2 - - -_PACK_I64 = struct.Struct('>q').pack -_FIRST_ELT = operator.itemgetter(0) - -USING_HBASE = False -NOW_MILLIS = int(1000 * time.time()) -TABLE_NAME = 'table-name' -ALT_TABLE_NAME = 'other-table' -TTL_FOR_TEST = 3 -COL_FAM1 = 'cf1' -COL_FAM2 = 'cf2' -COL_FAM3 = 'cf3' -FAMILIES = { - COL_FAM1: {'max_versions': 10}, - COL_FAM2: {'max_versions': 1, 'time_to_live': TTL_FOR_TEST}, - COL_FAM3: {}, # use defaults -} -ROW_KEY1 = 'row-key1' -ROW_KEY2 = 'row-key2a' -ROW_KEY3 = 'row-key2b' -COL1 = COL_FAM1 + ':qual1' -COL2 = COL_FAM1 + ':qual2' -COL3 = COL_FAM2 + ':qual1' -COL4 = COL_FAM3 + ':qual3' - - -class Config(object): - """Simple namespace for holding test globals.""" - - connection = None - table = None - - -def set_hbase_connection(): - from happybase import Connection - Config.connection = Connection() - - -def set_cloud_bigtable_connection(): - from gcloud_bigtable import client as client_mod - from gcloud_bigtable.happybase import Connection - - # NOTE: This assumes that the "gcloud-python" cluster in the - # "us-central1-c" zone already exists for this project. - # We are avoided checking "client.reload()" since the - # alpha version of grcpio does not correctly handle - # request timeouts. - client_mod.PROJECT_ENV_VAR = 'GCLOUD_TESTS_PROJECT_ID' - client = client_mod.Client(admin=True) - zone = 'us-central1-c' - cluster_id = 'gcloud-python' - cluster = client.cluster(zone, cluster_id) - - Config.connection = Connection(cluster=cluster) - - -def get_connection(): - if Config.connection is None: - if USING_HBASE: - set_hbase_connection() - else: - set_cloud_bigtable_connection() - return Config.connection - - -def get_table(): - if Config.table is None: - connection = get_connection() - Config.table = connection.table(TABLE_NAME) - return Config.table - - -def setUpModule(): - if Config.table is None: - connection = get_connection() - if TABLE_NAME not in connection.tables(): - connection.create_table(TABLE_NAME, FAMILIES) - - -def tearDownModule(): - connection = get_connection() - if not USING_HBASE: - connection.delete_table(TABLE_NAME) - connection.close() - - -class TestConnection(unittest2.TestCase): - - def test_create_and_delete_table(self): - connection = get_connection() - - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - connection.create_table(ALT_TABLE_NAME, {COL_FAM1: {}}) - self.assertTrue(ALT_TABLE_NAME in connection.tables()) - if USING_HBASE: - connection.delete_table(ALT_TABLE_NAME, disable=True) - else: - connection.delete_table(ALT_TABLE_NAME) - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - - def test_create_table_failure(self): - connection = get_connection() - - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - empty_families = {} - with self.assertRaises(ValueError): - connection.create_table(ALT_TABLE_NAME, empty_families) - self.assertFalse(ALT_TABLE_NAME in connection.tables()) - - -class BaseTableTest(unittest2.TestCase): - - def setUp(self): - self.rows_to_delete = [] - - def tearDown(self): - table = get_table() - for row_key in self.rows_to_delete: - table.delete(row_key) - - -class TestTable_families(BaseTableTest): - - def test_families(self): - table = get_table() - families = table.families() - - self.assertEqual(set(families.keys()), set(FAMILIES.keys())) - for col_fam, settings in FAMILIES.items(): - retrieved = families[col_fam] - for key, value in settings.items(): - if key == 'time_to_live' and USING_HBASE: - # The Thrift API fails to retrieve the TTL for some reason. - continue - self.assertEqual(retrieved[key], value) - - -class TestTable_row(BaseTableTest): - - def test_row_when_empty(self): - table = get_table() - row1 = table.row(ROW_KEY1) - row2 = table.row(ROW_KEY2) - - self.assertEqual(row1, {}) - self.assertEqual(row2, {}) - - def test_row_with_columns(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - row1_data = { - COL1: value1, - COL2: value2, - COL3: value3, - COL4: value4, - } - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - # Make sure the vanilla write succeeded. - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - # Pick out specific columns. - row1_diff_fams = table.row(ROW_KEY1, columns=[COL1, COL4]) - self.assertEqual(row1_diff_fams, {COL1: value1, COL4: value4}) - row1_single_col = table.row(ROW_KEY1, columns=[COL3]) - self.assertEqual(row1_single_col, {COL3: value3}) - row1_col_fam = table.row(ROW_KEY1, columns=[COL_FAM1]) - self.assertEqual(row1_col_fam, {COL1: value1, COL2: value2}) - row1_fam_qual_overlap1 = table.row(ROW_KEY1, columns=[COL1, COL_FAM1]) - self.assertEqual(row1_fam_qual_overlap1, {COL1: value1, COL2: value2}) - row1_fam_qual_overlap2 = table.row(ROW_KEY1, columns=[COL_FAM1, COL1]) - if USING_HBASE: - # NOTE: This behavior seems to be "incorrect" but that is how - # HappyBase / HBase works. - self.assertEqual(row1_fam_qual_overlap2, {COL1: value1}) - else: - self.assertEqual(row1_fam_qual_overlap2, - {COL1: value1, COL2: value2}) - row1_multiple_col_fams = table.row(ROW_KEY1, - columns=[COL_FAM1, COL_FAM2]) - self.assertEqual(row1_multiple_col_fams, - {COL1: value1, COL2: value2, COL3: value3}) - - def test_row_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL2: value2}) - table.put(ROW_KEY1, {COL3: value3}) - - # Make sure the vanilla write succeeded. - row1 = table.row(ROW_KEY1, include_timestamp=True) - ts1 = row1[COL1][1] - ts2 = row1[COL2][1] - ts3 = row1[COL3][1] - - expected_row = { - COL1: (value1, ts1), - COL2: (value2, ts2), - COL3: (value3, ts3), - } - self.assertEqual(row1, expected_row) - - # Make sure the timestamps are (strictly) ascending. - self.assertTrue(ts1 < ts2 < ts3) - - # Use timestamps to retrieve row. - first_two = table.row(ROW_KEY1, timestamp=ts2 + 1, - include_timestamp=True) - self.assertEqual(first_two, { - COL1: (value1, ts1), - COL2: (value2, ts2), - }) - first_one = table.row(ROW_KEY1, timestamp=ts2, - include_timestamp=True) - self.assertEqual(first_one, { - COL1: (value1, ts1), - }) - - -class TestTable_rows(BaseTableTest): - - def test_rows(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL1: value3} - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - table.put(ROW_KEY1, row1_data) - table.put(ROW_KEY2, row2_data) - - rows = table.rows([ROW_KEY1, ROW_KEY2]) - rows.sort(key=_FIRST_ELT) - - row1, row2 = rows - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - def test_rows_with_returned_timestamps(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL1: value3} - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - with table.batch() as batch: - batch.put(ROW_KEY1, row1_data) - batch.put(ROW_KEY2, row2_data) - - rows = table.rows([ROW_KEY1, ROW_KEY2], include_timestamp=True) - rows.sort(key=_FIRST_ELT) - - row1, row2 = rows - self.assertEqual(row1[0], ROW_KEY1) - self.assertEqual(row2[0], ROW_KEY2) - - # Drop the keys now that we have checked. - _, row1 = row1 - _, row2 = row2 - - ts = row1[COL1][1] - # All will have the same timestamp since we used batch. - expected_row1_result = {COL1: (value1, ts), COL2: (value2, ts)} - self.assertEqual(row1, expected_row1_result) - if USING_HBASE: - expected_row2_result = {COL1: (value3, ts)} - else: - # NOTE: Since Cloud Bigtable has no concept of batching, the - # server-side timestamps correspond to separate calls - # to row.commit(). We could circumvent this by manually - # using the local time and storing it on mutations before - # sending. - ts3 = row2[COL1][1] - expected_row2_result = {COL1: (value3, ts3)} - self.assertEqual(row2, expected_row2_result) - - def test_rows_with_columns(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL1: value3} - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - table.put(ROW_KEY1, row1_data) - table.put(ROW_KEY2, row2_data) - - # Filter a single column present in both rows. - rows_col1 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL1]) - rows_col1.sort(key=_FIRST_ELT) - row1, row2 = rows_col1 - self.assertEqual(row1, (ROW_KEY1, {COL1: value1})) - self.assertEqual(row2, (ROW_KEY2, {COL1: value3})) - - # Filter a column not present in one row. - rows_col2 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL2]) - self.assertEqual(rows_col2, [(ROW_KEY1, {COL2: value2})]) - - # Filter a column family. - rows_col_fam1 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL_FAM1]) - rows_col_fam1.sort(key=_FIRST_ELT) - row1, row2 = rows_col_fam1 - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - # Filter a column family with no entries. - rows_col_fam2 = table.rows([ROW_KEY1, ROW_KEY2], columns=[COL_FAM2]) - self.assertEqual(rows_col_fam2, []) - - # Filter a column family that overlaps with a column. - rows_col_fam_overlap1 = table.rows([ROW_KEY1, ROW_KEY2], - columns=[COL1, COL_FAM1]) - rows_col_fam_overlap1.sort(key=_FIRST_ELT) - row1, row2 = rows_col_fam_overlap1 - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - # Filter a column family that overlaps with a column (opposite order). - rows_col_fam_overlap2 = table.rows([ROW_KEY1, ROW_KEY2], - columns=[COL_FAM1, COL1]) - rows_col_fam_overlap2.sort(key=_FIRST_ELT) - row1, row2 = rows_col_fam_overlap2 - if USING_HBASE: - # NOTE: This behavior seems to be "incorrect" but that is how - # HappyBase / HBase works. - self.assertEqual(row1, (ROW_KEY1, {COL1: value1})) - else: - self.assertEqual(row1, (ROW_KEY1, row1_data)) - self.assertEqual(row2, (ROW_KEY2, row2_data)) - - def test_rows_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - - # Need to clean-up row1 and row2 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY2, {COL1: value2}) - table.put(ROW_KEY1, {COL2: value3}) - table.put(ROW_KEY1, {COL4: value4}) - - # Just grab the timestamps - rows = table.rows([ROW_KEY1, ROW_KEY2], include_timestamp=True) - rows.sort(key=_FIRST_ELT) - row1, row2 = rows - self.assertEqual(row1[0], ROW_KEY1) - self.assertEqual(row2[0], ROW_KEY2) - _, row1 = row1 - _, row2 = row2 - ts1 = row1[COL1][1] - ts2 = row2[COL1][1] - ts3 = row1[COL2][1] - ts4 = row1[COL4][1] - - # Make sure the timestamps are (strictly) ascending. - self.assertTrue(ts1 < ts2 < ts3 < ts4) - - # Rows before the third timestamp (assumes exclusive endpoint). - rows = table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts3, - include_timestamp=True) - rows.sort(key=_FIRST_ELT) - row1, row2 = rows - self.assertEqual(row1, (ROW_KEY1, {COL1: (value1, ts1)})) - self.assertEqual(row2, (ROW_KEY2, {COL1: (value2, ts2)})) - - # All writes (bump the exclusive endpoint by 1 millisecond). - rows = table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts4 + 1, - include_timestamp=True) - rows.sort(key=_FIRST_ELT) - row1, row2 = rows - row1_all_data = { - COL1: (value1, ts1), - COL2: (value3, ts3), - COL4: (value4, ts4), - } - self.assertEqual(row1, (ROW_KEY1, row1_all_data)) - self.assertEqual(row2, (ROW_KEY2, {COL1: (value2, ts2)})) - - # First three writes, restricted to column 2. - rows = table.rows([ROW_KEY1, ROW_KEY2], timestamp=ts4, - columns=[COL2], include_timestamp=True) - self.assertEqual(rows, [(ROW_KEY1, {COL2: (value3, ts3)})]) - - -class TestTable_cells(BaseTableTest): - - def test_cells(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL1: value2}) - table.put(ROW_KEY1, {COL1: value3}) - - # Check with no extra arguments. - all_values = table.cells(ROW_KEY1, COL1) - self.assertEqual(all_values, [value3, value2, value1]) - - # Check the timestamp on all the cells. - all_cells = table.cells(ROW_KEY1, COL1, include_timestamp=True) - self.assertEqual(len(all_cells), 3) - - ts3 = all_cells[0][1] - ts2 = all_cells[1][1] - ts1 = all_cells[2][1] - self.assertEqual(all_cells, - [(value3, ts3), (value2, ts2), (value1, ts1)]) - - # Limit to the two latest cells. - latest_two = table.cells(ROW_KEY1, COL1, include_timestamp=True, - versions=2) - self.assertEqual(latest_two, [(value3, ts3), (value2, ts2)]) - - # Limit to cells before the 2nd timestamp (inclusive). - first_two = table.cells(ROW_KEY1, COL1, include_timestamp=True, - timestamp=ts2 + 1) - self.assertEqual(first_two, [(value2, ts2), (value1, ts1)]) - - # Limit to cells before the 2nd timestamp (exclusive). - first_cell = table.cells(ROW_KEY1, COL1, include_timestamp=True, - timestamp=ts2) - self.assertEqual(first_cell, [(value1, ts1)]) - - -class TestTable_scan(BaseTableTest): - - def test_scan_when_empty(self): - table = get_table() - scan_result = list(table.scan()) - self.assertEqual(scan_result, []) - - def test_scan_single_row(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - scan_result = list(table.scan()) - self.assertEqual(scan_result, [(ROW_KEY1, row1_data)]) - - scan_result_cols = list(table.scan(columns=[COL1])) - self.assertEqual(scan_result_cols, [(ROW_KEY1, {COL1: value1})]) - - scan_result_ts = list(table.scan(include_timestamp=True)) - self.assertEqual(len(scan_result_ts), 1) - only_row = scan_result_ts[0] - self.assertEqual(only_row[0], ROW_KEY1) - row_values = only_row[1] - ts = row_values[COL1][1] - self.assertEqual(row_values, {COL1: (value1, ts), COL2: (value2, ts)}) - - if USING_HBASE: - scan_result_sorted = list(table.scan(sorted_columns=True)) - self.assertEqual(len(scan_result_sorted), 1) - only_row = scan_result_sorted[0] - self.assertEqual(only_row[0], ROW_KEY1) - row1_ordered = row1_data.items() - row1_ordered.sort(key=_FIRST_ELT) - self.assertEqual(only_row[1].items(), row1_ordered) - - def test_scan_filters(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - value5 = 'value5' - value6 = 'value6' - row1_data = {COL1: value1, COL2: value2} - row2_data = {COL2: value3, COL3: value4} - row3_data = {COL3: value5, COL4: value6} - - # Need to clean-up row1/2/3 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - self.rows_to_delete.append(ROW_KEY3) - table.put(ROW_KEY1, row1_data) - table.put(ROW_KEY2, row2_data) - table.put(ROW_KEY3, row3_data) - - # Basic scan (no filters) - scan_result = list(table.scan()) - self.assertEqual(scan_result, [ - (ROW_KEY1, row1_data), - (ROW_KEY2, row2_data), - (ROW_KEY3, row3_data), - ]) - - # Limit the size of the scan - scan_result = list(table.scan(limit=1)) - self.assertEqual(scan_result, [ - (ROW_KEY1, row1_data), - ]) - - # Scan with a row prefix. - prefix = ROW_KEY2[:-1] - self.assertEqual(prefix, ROW_KEY3[:-1]) - scan_result_prefixed = list(table.scan(row_prefix=prefix)) - self.assertEqual(scan_result_prefixed, [ - (ROW_KEY2, row2_data), - (ROW_KEY3, row3_data), - ]) - - # Make sure our keys are sorted in order - row_keys = [ROW_KEY1, ROW_KEY2, ROW_KEY3] - self.assertEqual(row_keys, sorted(row_keys)) - - # row_start alone (inclusive) - scan_result_row_start = list(table.scan(row_start=ROW_KEY2)) - self.assertEqual(scan_result_row_start, [ - (ROW_KEY2, row2_data), - (ROW_KEY3, row3_data), - ]) - - # row_stop alone (exclusive) - scan_result_row_stop = list(table.scan(row_stop=ROW_KEY2)) - self.assertEqual(scan_result_row_stop, [ - (ROW_KEY1, row1_data), - ]) - - # Both row_start and row_stop - scan_result_row_stop_and_start = list( - table.scan(row_start=ROW_KEY1, row_stop=ROW_KEY3)) - self.assertEqual(scan_result_row_stop_and_start, [ - (ROW_KEY1, row1_data), - (ROW_KEY2, row2_data), - ]) - - if USING_HBASE: - # Using a filter. - scan_result_filter = list(table.scan(filter='KeyOnlyFilter ()')) - self.assertEqual(scan_result_filter, [ - (ROW_KEY1, {COL1: '', COL2: ''}), # Keys only - (ROW_KEY2, {COL2: '', COL3: ''}), # Keys only - (ROW_KEY3, {COL3: '', COL4: ''}), # Keys only - ]) - - def test_scan_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - value4 = 'value4' - value5 = 'value5' - value6 = 'value6' - - # Need to clean-up row1/2/3 after. - self.rows_to_delete.append(ROW_KEY1) - self.rows_to_delete.append(ROW_KEY2) - self.rows_to_delete.append(ROW_KEY3) - table.put(ROW_KEY3, {COL4: value6}) - table.put(ROW_KEY2, {COL3: value4}) - table.put(ROW_KEY2, {COL2: value3}) - table.put(ROW_KEY1, {COL2: value2}) - table.put(ROW_KEY3, {COL3: value5}) - table.put(ROW_KEY1, {COL1: value1}) - - # Retrieve all the timestamps so we can filter with them. - scan_result = list(table.scan(include_timestamp=True)) - self.assertEqual(len(scan_result), 3) - row1, row2, row3 = scan_result - self.assertEqual(row1[0], ROW_KEY1) - self.assertEqual(row2[0], ROW_KEY2) - self.assertEqual(row3[0], ROW_KEY3) - - # Drop the keys now that we have checked. - _, row1 = row1 - _, row2 = row2 - _, row3 = row3 - - # These are numbered in order of insertion, **not** in - # the order of the values. - ts1 = row3[COL4][1] - ts2 = row2[COL3][1] - ts3 = row2[COL2][1] - ts4 = row1[COL2][1] - ts5 = row3[COL3][1] - ts6 = row1[COL1][1] - - self.assertEqual(row1, {COL1: (value1, ts6), COL2: (value2, ts4)}) - self.assertEqual(row2, {COL2: (value3, ts3), COL3: (value4, ts2)}) - self.assertEqual(row3, {COL3: (value5, ts5), COL4: (value6, ts1)}) - - # All cells before ts1 (exclusive) - scan_result_before_ts1 = list(table.scan(timestamp=ts1, - include_timestamp=True)) - self.assertEqual(scan_result_before_ts1, []) - - # All cells before ts2 (inclusive) - scan_result_before_ts2 = list(table.scan(timestamp=ts2 + 1, - include_timestamp=True)) - self.assertEqual(scan_result_before_ts2, [ - (ROW_KEY2, {COL3: (value4, ts2)}), - (ROW_KEY3, {COL4: (value6, ts1)}), - ]) - - # All cells before ts6 (exclusive) - scan_result_before_ts6 = list(table.scan(timestamp=ts6, - include_timestamp=True)) - self.assertEqual(scan_result_before_ts6, [ - (ROW_KEY1, {COL2: (value2, ts4)}), - (ROW_KEY2, {COL2: (value3, ts3), COL3: (value4, ts2)}), - (ROW_KEY3, {COL3: (value5, ts5), COL4: (value6, ts1)}), - ]) - - -class TestTable_put(BaseTableTest): - - def test_put(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - # Check again, but this time with timestamps. - row1 = table.row(ROW_KEY1, include_timestamp=True) - timestamp1 = row1[COL1][1] - timestamp2 = row1[COL2][1] - self.assertEqual(timestamp1, timestamp2) - - row1_data_with_timestamps = {COL1: (value1, timestamp1), - COL2: (value2, timestamp2)} - self.assertEqual(row1, row1_data_with_timestamps) - - @unittest2.skipIf(USING_HBASE, 'HBase fails to write with a timestamp') - def test_put_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - ts = NOW_MILLIS - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data, timestamp=ts) - - # Check again, but this time with timestamps. - row1 = table.row(ROW_KEY1, include_timestamp=True) - row1_data_with_timestamps = {COL1: (value1, ts), - COL2: (value2, ts)} - self.assertEqual(row1, row1_data_with_timestamps) - - @unittest2.skipIf(not USING_HBASE, ('Cloud Bigtable evictions do not seem ' - 'to occur immediately')) - def test_put_versions_restricted(self): - table = get_table() - families = table.families() - - chosen_fam = COL_FAM2 - self.assertEqual(families[chosen_fam]['max_versions'], 1) - chosen_col = COL3 - self.assertTrue(chosen_col.startswith(chosen_fam + ':')) - - value1 = 'value1' - value2 = 'value2' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {chosen_col: value1}) - - all_values_before = table.cells(ROW_KEY1, chosen_col, versions=2) - self.assertEqual(all_values_before, [value1]) - - # Putting another value should evict the first one. - table.put(ROW_KEY1, {chosen_col: value2}) - all_values_after = table.cells(ROW_KEY1, chosen_col, versions=2) - self.assertEqual(all_values_after, [value2]) - - @unittest2.skipIf(not USING_HBASE, ('Cloud Bigtable evictions do not seem ' - 'to occur immediately')) - def test_put_ttl_eviction(self): - table = get_table() - # The Thrift API fails to retrieve the TTL for some reason. - if USING_HBASE: - families = FAMILIES - else: - families = table.families() - - cell_tll = TTL_FOR_TEST - chosen_fam = COL_FAM2 - self.assertEqual(families[chosen_fam]['time_to_live'], cell_tll) - chosen_col = COL3 - self.assertTrue(chosen_col.startswith(chosen_fam + ':')) - - value1 = 'value1' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {chosen_col: value1}) - - all_values_before = table.cells(ROW_KEY1, chosen_col) - self.assertEqual(all_values_before, [value1]) - - # Make sure we don't sleep for a problematic length. - self.assertTrue(cell_tll < 10) - # Wait for time-to-live eviction to occur. - time.sleep(cell_tll + 0.5) - all_values_after = table.cells(ROW_KEY1, chosen_col) - self.assertEqual(all_values_after, []) - - -class TestTable_delete(BaseTableTest): - - def test_delete(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {}) - - def test_delete_with_columns(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {COL2: value2}) - - def test_delete_with_column_family(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - value3 = 'value3' - row1_data = {COL1: value1, COL2: value2, COL4: value3} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, row1_data) - - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL_FAM1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {COL4: value3}) - - def test_delete_with_columns_family_overlap(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - row1_data = {COL1: value1, COL2: value2} - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - # First go-around, use [COL_FAM1, COL1] - table.put(ROW_KEY1, row1_data) - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL_FAM1, COL1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {}) - - # Second go-around, use [COL1, COL_FAM1] - table.put(ROW_KEY1, row1_data) - row1 = table.row(ROW_KEY1) - self.assertEqual(row1, row1_data) - - table.delete(ROW_KEY1, columns=[COL1, COL_FAM1]) - row1_after = table.row(ROW_KEY1) - self.assertEqual(row1_after, {}) - - def test_delete_with_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL2: value2}) - - row1 = table.row(ROW_KEY1, include_timestamp=True) - ts1 = row1[COL1][1] - ts2 = row1[COL2][1] - - self.assertTrue(ts1 < ts2) - - if USING_HBASE: - # NOTE: HBase deletes use an inclusive timestamp at the endpoint. - table.delete(ROW_KEY1, timestamp=ts1 - 1) - else: - # NOTE: The Cloud Bigtable "Mutation.DeleteFromRow" mutation does - # not support timestamps. Even attempting to send one - # conditionally(via CheckAndMutateRowRequest) deletes the - # entire row. - # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp - # at the endpoint, but only because we fake this when - # creating Batch._delete_range. - table.delete(ROW_KEY1, columns=[COL1, COL2], timestamp=ts1 - 1) - row1_after_early_delete = table.row(ROW_KEY1, include_timestamp=True) - self.assertEqual(row1_after_early_delete, row1) - - if USING_HBASE: - # NOTE: HBase deletes use an inclusive timestamp at the endpoint. - table.delete(ROW_KEY1, timestamp=ts1) - else: - # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp - # at the endpoint, but only because we fake this when - # creating Batch._delete_range. - table.delete(ROW_KEY1, columns=[COL1, COL2], timestamp=ts1) - row1_after_incl_delete = table.row(ROW_KEY1, include_timestamp=True) - self.assertEqual(row1_after_incl_delete, {COL2: (value2, ts2)}) - - def test_delete_with_columns_and_timestamp(self): - table = get_table() - value1 = 'value1' - value2 = 'value2' - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - table.put(ROW_KEY1, {COL1: value1}) - table.put(ROW_KEY1, {COL2: value2}) - - row1 = table.row(ROW_KEY1, include_timestamp=True) - ts1 = row1[COL1][1] - ts2 = row1[COL2][1] - - # Delete with conditions that have no matches. - table.delete(ROW_KEY1, timestamp=ts1, columns=[COL2]) - row1_after_delete = table.row(ROW_KEY1, include_timestamp=True) - # NOTE: COL2 is still present since it occurs after ts1 and - # COL1 is still present since it is not in `columns`. - self.assertEqual(row1_after_delete, row1) - - # Delete with conditions that have no matches. - if USING_HBASE: - # NOTE: HBase deletes use an inclusive timestamp at the endpoint. - table.delete(ROW_KEY1, timestamp=ts1, columns=[COL_FAM1]) - else: - # NOTE: Cloud Bigtable can't use a timestamp with column families - # since "Mutation.DeleteFromFamily" does not include a - # timestamp range. - # NOTE: Cloud Bigtable deletes **ALSO** use an inclusive timestamp - # at the endpoint, but only because we fake this when - # creating Batch._delete_range. - table.delete(ROW_KEY1, timestamp=ts1, columns=[COL1, COL2]) - row1_delete_fam = table.row(ROW_KEY1, include_timestamp=True) - # NOTE: COL2 is still present since it occurs after ts1 and - # COL1 is still present since it is not in `columns`. - self.assertEqual(row1_delete_fam, {COL2: (value2, ts2)}) - - -class TestTableCounterMethods(BaseTableTest): - - def test_counter_get(self): - table = get_table() - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - initial_counter = table.counter_get(ROW_KEY1, COL1) - self.assertEqual(initial_counter, 0) - - # Check that the value is set (does not seem to occur on HBase). - if USING_HBASE: - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - else: - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), - {COL1: _PACK_I64(0)}) - - def test_counter_inc(self): - table = get_table() - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - initial_counter = table.counter_get(ROW_KEY1, COL1) - self.assertEqual(initial_counter, 0) - - inc_value = 10 - updated_counter = table.counter_inc(ROW_KEY1, COL1, value=inc_value) - self.assertEqual(updated_counter, inc_value) - - # Check that the value is set (does not seem to occur on HBase). - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), - {COL1: _PACK_I64(inc_value)}) - - def test_counter_dec(self): - table = get_table() - - # Need to clean-up row1 after. - self.rows_to_delete.append(ROW_KEY1) - - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), {}) - initial_counter = table.counter_get(ROW_KEY1, COL1) - self.assertEqual(initial_counter, 0) - - dec_value = 10 - updated_counter = table.counter_dec(ROW_KEY1, COL1, value=dec_value) - self.assertEqual(updated_counter, -dec_value) - - # Check that the value is set (does not seem to occur on HBase). - self.assertEqual(table.row(ROW_KEY1, columns=[COL1]), - {COL1: _PACK_I64(-dec_value)}) diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 8d64601..0000000 --- a/tox.ini +++ /dev/null @@ -1,61 +0,0 @@ -[tox] -envlist = - py27,cover,lint,docs - -[testenv] -commands = - {toxinidir}/scripts/nose_with_env.sh -deps = - nose - unittest2 -setenv = - PYTHONPATH = {toxinidir}/_fake_grpc - -[testenv:cover] -basepython = - python2.7 -commands = - {toxinidir}/scripts/nose_with_env.sh --with-xunit --with-xcoverage --cover-package=gcloud_bigtable --nocapture --cover-erase --cover-tests --cover-branches --cover-min-percentage=100 -deps = - {[testenv]deps} - coverage - nosexcover - -[pep8] -exclude = gcloud_bigtable/_generated/*,docs/conf.py -verbose = 1 - -[testenv:lint] -basepython = - python2.7 -commands = - {toxinidir}/scripts/pep8_on_repo.sh - python run_pylint.py -deps = - pep8 - -ehg+https://bitbucket.org/logilab/pylint@33e334be064c#egg=pylint - unittest2 -passenv = GCLOUD_* - -[testenv:system-tests] -basepython = - python2.7 -commands = - {toxinidir}/scripts/nose_with_env.sh -v {toxinidir}/system_tests/run.py - {toxinidir}/scripts/nose_with_env.sh -v {toxinidir}/system_tests/run_happybase.py -deps = - grpcio==0.10.0a0 - nose - unittest2 -passenv = GOOGLE_* GCLOUD_* GRPC_TRACE -setenv = - -[testenv:docs] -basepython = - python2.7 -commands = - python -c "import shutil; shutil.rmtree('docs/_build', ignore_errors=True)" - {toxinidir}/scripts/sphinx_with_env.sh -W -b html -d docs/_build/doctrees docs docs/_build/html -deps = - Sphinx -passenv = SPHINX_RELEASE